60#include "llvm/IR/IntrinsicsAArch64.h"
95#define DEBUG_TYPE "aarch64-lower"
106 cl::desc(
"Allow AArch64 Local Dynamic TLS code generation"),
111 cl::desc(
"Enable AArch64 logical imm instruction "
121 cl::desc(
"Combine extends of AArch64 masked "
122 "gather intrinsics"),
154 switch (EC.getKnownMinValue()) {
170 "Expected scalable predicate vector type!");
192 "Expected legal vector type!");
280 if (Subtarget->
hasSVE()) {
308 if (useSVEForFixedLengthVectorVT(VT))
312 if (useSVEForFixedLengthVectorVT(VT))
730#define LCALLNAMES(A, B, N) \
731 setLibcallName(A##N##_RELAX, #B #N "_relax"); \
732 setLibcallName(A##N##_ACQ, #B #N "_acq"); \
733 setLibcallName(A##N##_REL, #B #N "_rel"); \
734 setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
735#define LCALLNAME4(A, B) \
736 LCALLNAMES(A, B, 1) \
737 LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
738#define LCALLNAME5(A, B) \
739 LCALLNAMES(A, B, 1) \
740 LCALLNAMES(A, B, 2) \
741 LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
1092 if (Subtarget->
hasSVE()) {
1203 if (useSVEForFixedLengthVectorVT(VT))
1204 addTypeForFixedLengthSVE(VT);
1206 if (useSVEForFixedLengthVectorVT(VT))
1207 addTypeForFixedLengthSVE(VT);
1347 for (
unsigned Opcode :
1360void AArch64TargetLowering::addTypeForFixedLengthSVE(
MVT VT) {
1431void AArch64TargetLowering::addDRTypeForNEON(
MVT VT) {
1436void AArch64TargetLowering::addQRTypeForNEON(
MVT VT) {
1455 uint64_t Mask = ((uint64_t)(-1LL) >> (64 -
Size)),
OrigMask = Mask;
1459 if (Imm == 0 || Imm == Mask ||
1463 unsigned EltSize =
Size;
1513 while (EltSize <
Size) {
1520 "demanded bits should never be altered");
1524 EVT VT = Op.getValueType();
1531 New =
TLO.DAG.getNode(Op.getOpcode(),
DL, VT, Op.getOperand(0),
1542 return TLO.CombineTo(Op, New);
1555 EVT VT = Op.getValueType();
1561 "i32 or i64 is expected after legalization.");
1568 switch (Op.getOpcode()) {
1572 NewOpc =
Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1575 NewOpc =
Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1578 NewOpc =
Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1584 uint64_t Imm =
C->getZExtValue();
1593 switch (Op.getOpcode()) {
1616 case Intrinsic::aarch64_ldaxr:
1617 case Intrinsic::aarch64_ldxr: {
1633 case Intrinsic::aarch64_neon_umaxv:
1634 case Intrinsic::aarch64_neon_uminv: {
1639 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1696 Ty.getSizeInBytes() != 16 ||
1719#define MAKE_CASE(V) \
2021 Register DestReg =
MI.getOperand(0).getReg();
2024 unsigned CondCode =
MI.getOperand(3).getImm();
2056 MI.eraseFromParent();
2064 "SEH does not use catchret!");
2070 switch (
MI.getOpcode()) {
2077 case AArch64::F128CSEL:
2080 case TargetOpcode::STACKMAP:
2081 case TargetOpcode::PATCHPOINT:
2082 case TargetOpcode::STATEPOINT:
2085 case AArch64::CATCHRET:
2254 bool IsLegal = (
C >> 12 == 0) || ((
C & 0xFFFULL) == 0 &&
C >> 24 == 0);
2256 <<
" legal: " << (
IsLegal ?
"yes\n" :
"no\n"));
2278 EVT VT = LHS.getValueType();
2280 assert(VT !=
MVT::f16 &&
"Lowering of strict fp16 not yet implemented");
2288 EVT VT = LHS.getValueType();
2308 if (
isCMN(RHS, CC)) {
2311 RHS = RHS.getOperand(1);
2312 }
else if (
isCMN(LHS, CC)) {
2316 LHS = LHS.getOperand(1);
2331 return LHS.getValue(1);
2397 unsigned Opcode = 0;
2401 if (LHS.getValueType().isFloatingPoint()) {
2408 }
else if (RHS.getOpcode() ==
ISD::SUB) {
2413 RHS = RHS.getOperand(1);
2442 unsigned Depth = 0) {
2511 bool isInteger = LHS.getValueType().isInteger();
2513 CC = getSetCCInverse(CC, LHS.getValueType());
2519 assert(LHS.getValueType().isFloatingPoint());
2551 assert(
ValidL &&
"Valid conjunction/disjunction tree");
2558 assert(
ValidR &&
"Valid conjunction/disjunction tree");
2590 assert(Opcode ==
ISD::AND &&
"Valid conjunction/disjunction tree");
2591 assert(!Negate &&
"Valid conjunction/disjunction tree");
2635 uint64_t Mask =
MaskCst->getZExtValue();
2636 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2642 if (!Op.hasOneUse())
2648 unsigned Opc = Op.getOpcode();
2651 uint64_t Shift =
ShiftCst->getZExtValue();
2653 return (Shift <= 4) ? 2 : 1;
2654 EVT VT = Op.getValueType();
2666 EVT VT = RHS.getValueType();
2667 uint64_t
C =
RHSC->getZExtValue();
2675 if ((VT ==
MVT::i32 &&
C != 0x80000000 &&
2677 (VT ==
MVT::i64 &&
C != 0x80000000ULL &&
2764 LHS.getNode()->hasNUsesOfValue(1, 0)) {
2771 RHS.getValueType()),
2777 if (!Cmp && (
RHSC->isNullValue() ||
RHSC->isOne())) {
2793static std::pair<SDValue, SDValue>
2796 "Unsupported value type");
2799 SDValue LHS = Op.getOperand(0);
2800 SDValue RHS = Op.getOperand(1);
2802 switch (Op.getOpcode()) {
2825 bool IsSigned = Op.getOpcode() ==
ISD::SMULO;
2826 if (Op.getValueType() ==
MVT::i32) {
2874 assert(Op.getValueType() ==
MVT::i64 &&
"Expected an i64 value type");
2909 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
2910 return LowerToScalableOp(Op, DAG);
2969 if (
CTVal->isAllOnesValue() &&
CFVal->isNullValue()) {
2976 if (
CTVal->isNullValue() &&
CFVal->isAllOnesValue()) {
2992 EVT VT = Op.getValueType();
3002 switch (Op.getOpcode()) {
3022 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3023 return DAG.
getNode(Opc,
SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3076 unsigned PrfOp = (IsWrite << 4) |
3086 if (
Op.getValueType().isScalableVector())
3095 if (
Op.getValueType().isScalableVector())
3104 if (useSVEForFixedLengthVectorVT(
SrcVT))
3119 EVT InVT =
Op.getOperand(0).getValueType();
3120 EVT VT =
Op.getValueType();
3126 return LowerToPredicatedOp(Op, DAG, Opcode);
3137 Op.getOpcode(), dl,
Op.getValueType(),
3146 DAG.
getNode(
Op.getOpcode(), dl,
InVT.changeVectorElementTypeToInteger(),
3157 return DAG.
getNode(
Op.getOpcode(), dl, VT, Ext);
3169 if (
SrcVal.getValueType().isVector())
3170 return LowerVectorFP_TO_INT(Op, DAG);
3174 assert(!
IsStrict &&
"Lowering of strict fp16 not yet implemented");
3177 Op.getOpcode(), dl,
Op.getValueType(),
3194 EVT VT =
Op.getValueType();
3198 unsigned Opc =
Op.getOpcode();
3207 return DAG.
getNode(Opc, dl, VT, In);
3212 return LowerToPredicatedOp(Op, DAG, Opcode);
3220 InVT.getVectorNumElements());
3229 return DAG.
getNode(Opc, dl, VT, In);
3237 if (
Op.getValueType().isVector())
3238 return LowerVectorINT_TO_FP(Op, DAG);
3246 assert(!
IsStrict &&
"Lowering of strict fp16 not yet implemented");
3271 EVT ArgVT =
Arg.getValueType();
3279 Entry.IsSExt =
false;
3280 Entry.IsZExt =
false;
3281 Args.push_back(Entry);
3284 : RTLIB::SINCOS_STRET_F32;
3316 if (
OrigVT.getSizeInBits() >= 64)
3319 assert(
OrigVT.isSimple() &&
"Expecting a simple value type");
3351 EVT VT =
N->getValueType(0);
3384 EVT VT =
N->getValueType(0);
3413 unsigned Opcode =
N->getOpcode();
3416 SDNode *
N1 =
N->getOperand(1).getNode();
3424 unsigned Opcode =
N->getOpcode();
3427 SDNode *
N1 =
N->getOperand(1).getNode();
3458 EVT VT =
Op.getValueType();
3469 "unexpected type for custom-lowering ISD::MUL");
3470 SDNode *N0 =
Op.getOperand(0).getNode();
3517 "unexpected types for extended operands to VMULL");
3539SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(
SDValue Op,
3545 case Intrinsic::thread_pointer: {
3549 case Intrinsic::aarch64_neon_abs: {
3550 EVT Ty =
Op.getValueType();
3556 }
else if (Ty.isVector() && Ty.isInteger() &&
isTypeLegal(Ty)) {
3562 case Intrinsic::aarch64_neon_smax:
3564 Op.getOperand(1),
Op.getOperand(2));
3565 case Intrinsic::aarch64_neon_umax:
3567 Op.getOperand(1),
Op.getOperand(2));
3568 case Intrinsic::aarch64_neon_smin:
3570 Op.getOperand(1),
Op.getOperand(2));
3571 case Intrinsic::aarch64_neon_umin:
3573 Op.getOperand(1),
Op.getOperand(2));
3575 case Intrinsic::aarch64_sve_sunpkhi:
3578 case Intrinsic::aarch64_sve_sunpklo:
3581 case Intrinsic::aarch64_sve_uunpkhi:
3584 case Intrinsic::aarch64_sve_uunpklo:
3587 case Intrinsic::aarch64_sve_clasta_n:
3589 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3590 case Intrinsic::aarch64_sve_clastb_n:
3592 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
3593 case Intrinsic::aarch64_sve_lasta:
3595 Op.getOperand(1),
Op.getOperand(2));
3596 case Intrinsic::aarch64_sve_lastb:
3598 Op.getOperand(1),
Op.getOperand(2));
3599 case Intrinsic::aarch64_sve_rev:
3602 case Intrinsic::aarch64_sve_tbl:
3604 Op.getOperand(1),
Op.getOperand(2));
3605 case Intrinsic::aarch64_sve_trn1:
3607 Op.getOperand(1),
Op.getOperand(2));
3608 case Intrinsic::aarch64_sve_trn2:
3610 Op.getOperand(1),
Op.getOperand(2));
3611 case Intrinsic::aarch64_sve_uzp1:
3613 Op.getOperand(1),
Op.getOperand(2));
3614 case Intrinsic::aarch64_sve_uzp2:
3616 Op.getOperand(1),
Op.getOperand(2));
3617 case Intrinsic::aarch64_sve_zip1:
3619 Op.getOperand(1),
Op.getOperand(2));
3620 case Intrinsic::aarch64_sve_zip2:
3622 Op.getOperand(1),
Op.getOperand(2));
3623 case Intrinsic::aarch64_sve_ptrue:
3626 case Intrinsic::aarch64_sve_clz:
3628 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3629 case Intrinsic::aarch64_sve_cnt: {
3632 if (
Data.getValueType().isFloatingPoint())
3635 Op.getOperand(2),
Data,
Op.getOperand(1));
3637 case Intrinsic::aarch64_sve_dupq_lane:
3638 return LowerDUPQLane(Op, DAG);
3639 case Intrinsic::aarch64_sve_convert_from_svbool:
3642 case Intrinsic::aarch64_sve_fneg:
3644 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3645 case Intrinsic::aarch64_sve_frintp:
3647 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3648 case Intrinsic::aarch64_sve_frintm:
3650 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3651 case Intrinsic::aarch64_sve_frinti:
3653 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3654 case Intrinsic::aarch64_sve_frintx:
3656 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3657 case Intrinsic::aarch64_sve_frinta:
3659 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3660 case Intrinsic::aarch64_sve_frintn:
3662 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3663 case Intrinsic::aarch64_sve_frintz:
3665 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3666 case Intrinsic::aarch64_sve_ucvtf:
3668 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3670 case Intrinsic::aarch64_sve_scvtf:
3672 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3674 case Intrinsic::aarch64_sve_fcvtzu:
3676 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3678 case Intrinsic::aarch64_sve_fcvtzs:
3680 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3682 case Intrinsic::aarch64_sve_fsqrt:
3684 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3685 case Intrinsic::aarch64_sve_frecpx:
3687 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3688 case Intrinsic::aarch64_sve_fabs:
3690 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3691 case Intrinsic::aarch64_sve_abs:
3693 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3694 case Intrinsic::aarch64_sve_neg:
3696 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3697 case Intrinsic::aarch64_sve_convert_to_svbool: {
3699 EVT InVT =
Op.getOperand(1).getValueType();
3703 return Op.getOperand(1);
3713 case Intrinsic::aarch64_sve_insr: {
3720 Op.getOperand(1), Scalar);
3722 case Intrinsic::aarch64_sve_rbit:
3724 Op.getValueType(),
Op.getOperand(2),
Op.getOperand(3),
3726 case Intrinsic::aarch64_sve_revb:
3728 Op.getOperand(2),
Op.getOperand(3),
Op.getOperand(1));
3729 case Intrinsic::aarch64_sve_sxtb:
3732 Op.getOperand(2),
Op.getOperand(3),
3735 case Intrinsic::aarch64_sve_sxth:
3738 Op.getOperand(2),
Op.getOperand(3),
3741 case Intrinsic::aarch64_sve_sxtw:
3744 Op.getOperand(2),
Op.getOperand(3),
3747 case Intrinsic::aarch64_sve_uxtb:
3750 Op.getOperand(2),
Op.getOperand(3),
3753 case Intrinsic::aarch64_sve_uxth:
3756 Op.getOperand(2),
Op.getOperand(3),
3759 case Intrinsic::aarch64_sve_uxtw:
3762 Op.getOperand(2),
Op.getOperand(3),
3766 case Intrinsic::localaddress: {
3769 unsigned Reg =
RegInfo->getLocalAddressRegister(MF);
3771 Op.getSimpleValueType());
3774 case Intrinsic::eh_recoverfp: {
3784 "llvm.eh.recoverfp must take a function as the first argument");
3788 case Intrinsic::aarch64_neon_vsri:
3789 case Intrinsic::aarch64_neon_vsli: {
3790 EVT Ty =
Op.getValueType();
3795 assert(
Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
3799 return DAG.
getNode(Opcode, dl, Ty,
Op.getOperand(1),
Op.getOperand(2),
3803 case Intrinsic::aarch64_neon_srhadd:
3804 case Intrinsic::aarch64_neon_urhadd:
3805 case Intrinsic::aarch64_neon_shadd:
3806 case Intrinsic::aarch64_neon_uhadd: {
3808 IntNo == Intrinsic::aarch64_neon_shadd);
3810 IntNo == Intrinsic::aarch64_neon_urhadd);
3814 return DAG.
getNode(Opcode, dl,
Op.getValueType(),
Op.getOperand(1),
3818 case Intrinsic::aarch64_neon_uabd: {
3820 Op.getOperand(1),
Op.getOperand(2));
3822 case Intrinsic::aarch64_neon_sabd: {
3824 Op.getOperand(1),
Op.getOperand(2));
3829bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(
EVT VT)
const {
3837bool AArch64TargetLowering::isVectorLoadExtDesirable(
SDValue ExtVal)
const {
3838 return ExtVal.getValueType().isScalableVector();
3842 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
3843 {std::make_tuple(
false,
false,
false),
3845 {std::make_tuple(
false,
false,
true),
3847 {std::make_tuple(
false,
true,
false),
3849 {std::make_tuple(
false,
true,
true),
3851 {std::make_tuple(
true,
false,
false),
3853 {std::make_tuple(
true,
false,
true),
3855 {std::make_tuple(
true,
true,
false),
3857 {std::make_tuple(
true,
true,
true),
3861 return AddrModes.find(Key)->second;
3865 std::map<std::tuple<bool, bool, bool>,
unsigned> AddrModes = {
3866 {std::make_tuple(
false,
false,
false),
3868 {std::make_tuple(
false,
false,
true),
3870 {std::make_tuple(
false,
true,
false),
3872 {std::make_tuple(
false,
true,
true),
3874 {std::make_tuple(
true,
false,
false),
3876 {std::make_tuple(
true,
false,
true),
3878 {std::make_tuple(
true,
true,
false),
3880 {std::make_tuple(
true,
true,
true),
3884 return AddrModes.find(Key)->second;
3910 unsigned Opcode = Index.getOpcode();
3919 if (!Mask || Mask->getZExtValue() != 0xFFFFFFFF)
3949 Index = Index->getOperand(0);
3970 Index = Index->getOperand(0);
3976 BasePtr = Index->getOperand(0);
3984 assert(
MGT &&
"Can only custom lower gather load nodes");
4000 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4021 Index = Index.getOperand(0);
4045 assert(
MSC &&
"Can only custom lower scatter store nodes");
4060 Index.getSimpleValueType().getVectorElementType() ==
MVT::i32;
4086 return DAG.
getNode(Opcode,
DL, VTs, Ops);
4107 {Undef, Undef, Undef, Undef});
4118 ST->getBasePtr(), ST->getMemOperand());
4136 if (useSVEForFixedLengthVectorVT(VT))
4137 return LowerFixedLengthVectorStoreToSVE(Op, DAG);
4139 unsigned AS =
StoreNode->getAddressSpace();
4141 if (Alignment <
MemVT.getStoreSize() &&
4155 if (
StoreNode->isNonTemporal() &&
MemVT.getSizeInBits() == 256u &&
4157 ((
MemVT.getScalarSizeInBits() == 8u ||
4158 MemVT.getScalarSizeInBits() == 16u ||
4159 MemVT.getScalarSizeInBits() == 32u ||
4160 MemVT.getScalarSizeInBits() == 64u))) {
4172 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4186 {StoreNode->getChain(), Lo, Hi, StoreNode->getBasePtr()},
4196 MVT VT =
Op.getSimpleValueType();
4218 switch (Op.getOpcode()) {
4225 return LowerGlobalAddress(Op, DAG);
4227 return LowerGlobalTLSAddress(Op, DAG);
4231 return LowerSETCC(Op, DAG);
4233 return LowerBR_CC(Op, DAG);
4235 return LowerSELECT(Op, DAG);
4237 return LowerSELECT_CC(Op, DAG);
4239 return LowerJumpTable(Op, DAG);
4241 return LowerBR_JT(Op, DAG);
4243 return LowerConstantPool(Op, DAG);
4245 return LowerBlockAddress(Op, DAG);
4247 return LowerVASTART(Op, DAG);
4249 return LowerVACOPY(Op, DAG);
4251 return LowerVAARG(Op, DAG);
4296 return LowerFP_ROUND(Op, DAG);
4298 return LowerFP_EXTEND(Op, DAG);
4300 return LowerFRAMEADDR(Op, DAG);
4302 return LowerSPONENTRY(Op, DAG);
4304 return LowerRETURNADDR(Op, DAG);
4306 return LowerADDROFRETURNADDR(Op, DAG);
4308 return LowerCONCAT_VECTORS(Op, DAG);
4310 return LowerINSERT_VECTOR_ELT(Op, DAG);
4312 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
4314 return LowerBUILD_VECTOR(Op, DAG);
4316 return LowerVECTOR_SHUFFLE(Op, DAG);
4318 return LowerSPLAT_VECTOR(Op, DAG);
4320 return LowerEXTRACT_SUBVECTOR(Op, DAG);
4322 return LowerINSERT_SUBVECTOR(Op, DAG);
4325 return LowerDIV(Op, DAG);
4341 return LowerVectorSRA_SRL_SHL(Op, DAG);
4343 return LowerShiftLeftParts(Op, DAG);
4346 return LowerShiftRightParts(Op, DAG);
4348 return LowerCTPOP(Op, DAG);
4350 return LowerFCOPYSIGN(Op, DAG);
4352 return LowerVectorOR(Op, DAG);
4354 return LowerXOR(Op, DAG);
4361 return LowerINT_TO_FP(Op, DAG);
4366 return LowerFP_TO_INT(Op, DAG);
4368 return LowerFSINCOS(Op, DAG);
4370 return LowerFLT_ROUNDS_(Op, DAG);
4372 return LowerMUL(Op, DAG);
4374 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4376 return LowerSTORE(Op, DAG);
4378 return LowerMGATHER(Op, DAG);
4380 return LowerMSCATTER(Op, DAG);
4382 return LowerVECREDUCE_SEQ_FADD(Op, DAG);
4394 return LowerVECREDUCE(Op, DAG);
4396 return LowerATOMIC_LOAD_SUB(Op, DAG);
4398 return LowerATOMIC_LOAD_AND(Op, DAG);
4400 return LowerDYNAMIC_STACKALLOC(Op, DAG);
4402 return LowerVSCALE(Op, DAG);
4406 return LowerFixedLengthVectorIntExtendToSVE(Op, DAG);
4415 return LowerToPredicatedOp(Op, DAG,
4419 return LowerTRUNCATE(Op, DAG);
4421 if (useSVEForFixedLengthVectorVT(Op.getValueType()))
4422 return LowerFixedLengthVectorLoadToSVE(Op, DAG);
4427 return LowerToScalableOp(Op, DAG);
4435 return LowerFixedLengthVectorSelectToSVE(Op, DAG);
4437 return LowerABS(Op, DAG);
4447 return LowerCTTZ(Op, DAG);
4455bool AArch64TargetLowering::useSVEForFixedLengthVectorVT(
4506 bool IsVarArg)
const {
4543SDValue AArch64TargetLowering::LowerFormalArguments(
4563 unsigned NumArgs = Ins.size();
4566 for (
unsigned i = 0;
i != NumArgs; ++
i) {
4567 MVT ValVT = Ins[
i].VT;
4568 if (Ins[
i].isOrigArg()) {
4585 assert(!Res &&
"Call operand has unhandled type");
4590 for (
unsigned i = 0, e =
Ins.size();
i !=
e; ++
i) {
4593 if (Ins[
i].Flags.isByVal()) {
4597 int Size =
Ins[
i].Flags.getByValSize();
4598 unsigned NumRegs = (
Size + 7) / 8;
4611 if (
VA.isRegLoc()) {
4613 EVT RegVT =
VA.getLocVT();
4617 RC = &AArch64::GPR32RegClass;
4619 RC = &AArch64::GPR64RegClass;
4621 RC = &AArch64::FPR16RegClass;
4623 RC = &AArch64::FPR32RegClass;
4625 RC = &AArch64::FPR64RegClass;
4627 RC = &AArch64::FPR128RegClass;
4630 RC = &AArch64::PPRRegClass;
4632 RC = &AArch64::ZPRRegClass;
4643 switch (
VA.getLocInfo()) {
4649 assert(
VA.getValVT().isScalableVector() &&
4650 "Only scalable vectors can be passed indirectly");
4666 assert(
VA.isMemLoc() &&
"CCValAssign is neither reg nor mem");
4669 ?
VA.getLocVT().getSizeInBits()
4670 :
VA.getValVT().getSizeInBits()) / 8;
4674 !
Ins[
i].Flags.isInConsecutiveRegs())
4686 switch (
VA.getLocInfo()) {
4694 assert(
VA.getValVT().isScalableVector() &&
4695 "Only scalable vectors can be passed indirectly");
4710 ExtType,
DL,
VA.getLocVT(), Chain,
FIN,
4717 assert(
VA.getValVT().isScalableVector() &&
4718 "Only scalable vectors can be passed indirectly");
4720 uint64_t
PartSize =
VA.getValVT().getStoreSize().getKnownMinSize();
4722 if (Ins[
i].Flags.isInConsecutiveRegs()) {
4723 assert(!Ins[
i].Flags.isInConsecutiveRegsLast());
4724 while (!Ins[
i +
NumParts - 1].Flags.isInConsecutiveRegsLast())
4735 InVals.push_back(ArgValue);
4753 InVals.push_back(ArgValue);
4767 saveVarArgRegisters(CCInfo, DAG,
DL, Chain);
4771 unsigned StackOffset = CCInfo.getNextStackOffset();
4787 if (!CCInfo.isAllocated(AArch64::X8)) {
4798 for (
unsigned I = 0,
E =
Ins.size();
I !=
E; ++
I) {
4799 if (Ins[
I].Flags.isInReg()) {
4816 if (DoesCalleeRestoreStack(CallConv,
TailCallOpt)) {
4840void AArch64TargetLowering::saveVarArgRegisters(
CCState &CCInfo,
4853 AArch64::X3, AArch64::X4, AArch64::X5,
4854 AArch64::X6, AArch64::X7 };
4891 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
4892 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
4926SDValue AArch64TargetLowering::LowerCallResult(
4940 for (
unsigned i = 0;
i !=
RVLocs.size(); ++
i) {
4947 "unexpected return calling convention register assignment");
4963 switch (
VA.getLocInfo()) {
4982 InVals.push_back(Val);
5006bool AArch64TargetLowering::isEligibleForTailCallOptimization(
5041 if (
i->hasByValAttr())
5050 if (
i->hasInRegAttr())
5068 (!
TT.isOSWindows() ||
TT.isOSBinFormatELF() ||
TT.isOSBinFormatMachO()))
5079 "Unexpected variadic calling convention");
5082 if (isVarArg && !Outs.empty()) {
5133 A.getValVT().isScalableVector()) &&
5134 "Expected value to be scalable");
5170 if (FI->getIndex() < 0) {
5192AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
5201 bool &IsTailCall = CLI.IsTailCall;
5203 bool IsVarArg = CLI.IsVarArg;
5220 return In.VT.isScalableVector();
5229 IsTailCall = isEligibleForTailCallOptimization(
5230 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
5231 if (!IsTailCall && CLI.CB && CLI.CB->isMustTailCall())
5233 "site marked musttail");
5252 unsigned NumArgs = Outs.size();
5254 for (
unsigned i = 0;
i != NumArgs; ++
i) {
5255 MVT ArgVT = Outs[
i].VT;
5258 "currently not supported");
5264 assert(!Res &&
"Call operand has unhandled type");
5274 unsigned NumArgs = Outs.size();
5275 for (
unsigned i = 0;
i != NumArgs; ++
i) {
5276 MVT ValVT = Outs[
i].VT;
5279 CLI.getArgs()[Outs[
i].OrigArgIndex].Ty,
5291 assert(!Res &&
"Call operand has unhandled type");
5317 NumBytes =
alignTo(NumBytes, 16);
5329 assert(FPDiff % 16 == 0 &&
"unaligned stack on tail call");
5345 if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
5355 for (
unsigned i = 0, e = Outs.size();
i !=
e; ++
i) {
5361 switch (
VA.getLocInfo()) {
5396 assert(
VA.getValVT().isScalableVector() &&
5397 "Only scalable vectors can be passed indirectly");
5399 uint64_t StoreSize =
VA.getValVT().getStoreSize().getKnownMinSize();
5402 if (Outs[
i].Flags.isInConsecutiveRegs()) {
5403 assert(!Outs[
i].Flags.isInConsecutiveRegsLast());
5404 while (!Outs[
i +
NumParts - 1].Flags.isInConsecutiveRegsLast())
5424 Chain = DAG.
getStore(Chain,
DL, OutVals[
i], Ptr, MPI);
5445 if (
VA.isRegLoc()) {
5446 if (
i == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
5449 "unexpected calling convention register assignment");
5451 "unexpected use of 'returned'");
5454 if (RegsUsed.count(
VA.getLocReg())) {
5461 [=](
const std::pair<unsigned, SDValue> &
Elt) {
5462 return Elt.first ==
VA.getLocReg();
5470 return ArgReg.Reg ==
VA.getLocReg();
5474 RegsUsed.insert(
VA.getLocReg());
5477 CSInfo.emplace_back(
VA.getLocReg(),
i);
5490 OpSize =
VA.getLocVT().getFixedSizeInBits();
5492 OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
5493 :
VA.getValVT().getSizeInBits();
5496 !Flags.isInConsecutiveRegs()) {
5516 Chain = addTokenForArgument(Chain, DAG, MF.
getFrameInfo(), FI);
5525 if (Outs[
i].Flags.isByVal()) {
5530 Outs[
i].Flags.getNonZeroByValAlign(),
5565 auto GV =
G->getGlobal();
5578 const char *Sym = S->getSymbol();
5582 const char *Sym = S->getSymbol();
5597 std::vector<SDValue> Ops;
5598 Ops.push_back(Chain);
5611 Ops.push_back(DAG.getRegister(
RegToPass.first,
5619 Mask =
TRI->getThisReturnPreservedMask(MF, CallConv);
5622 Mask =
TRI->getCallPreservedMask(MF, CallConv);
5625 Mask =
TRI->getCallPreservedMask(MF, CallConv);
5628 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
5630 if (
TRI->isAnyArgRegReserved(MF))
5631 TRI->emitReservedArgRegCallError(MF);
5633 assert(Mask &&
"Missing call preserved mask for calling convention");
5654 if (CLI.CB && CLI.CB->hasRetAttr(
"rv_marker")) {
5655 assert(!IsTailCall &&
"tail calls cannot be marked with rv_marker");
5676 return LowerCallResult(Chain,
InFlag, CallConv, IsVarArg, Ins,
DL, DAG,
5681bool AArch64TargetLowering::CanLowerReturn(
5712 assert(
VA.isRegLoc() &&
"Can only return in registers!");
5715 switch (
VA.getLocInfo()) {
5742 if (RegsUsed.count(
VA.getLocReg())) {
5745 return Elt.first ==
VA.getLocReg();
5750 RegsUsed.insert(
VA.getLocReg());
5755 for (
auto &RetVal :
RetVals) {
5756 Chain = DAG.
getCopyToReg(Chain,
DL, RetVal.first, RetVal.second, Flag);
5759 DAG.
getRegister(RetVal.first, RetVal.second.getValueType()));
5783 if (AArch64::GPR64RegClass.
contains(*
I))
5785 else if (AArch64::FPR64RegClass.
contains(*
I))
5807 unsigned Flag)
const {
5809 N->getOffset(), Flag);
5814 unsigned Flag)
const {
5820 unsigned Flag)
const {
5822 N->getOffset(), Flag);
5827 unsigned Flag)
const {
5832template <
class NodeTy>
5834 unsigned Flags)
const {
5845template <
class NodeTy>
5847 unsigned Flags)
const {
5861template <
class NodeTy>
5863 unsigned Flags)
const {
5875template <
class NodeTy>
5877 unsigned Flags)
const {
5881 SDValue Sym = getTargetNode(
N, Ty, DAG, Flags);
5893 "unexpected offset in global node");
5946AArch64TargetLowering::LowerDarwinGlobalTLSAddress(
SDValue Op,
5949 "This function expects a Darwin target");
6120AArch64TargetLowering::LowerELFGlobalTLSAddress(
SDValue Op,
6136 "in local exec TLS model");
6207AArch64TargetLowering::LowerWindowsGlobalTLSAddress(
SDValue Op,
6272 return LowerDarwinGlobalTLSAddress(Op, DAG);
6274 return LowerELFGlobalTLSAddress(Op, DAG);
6276 return LowerWindowsGlobalTLSAddress(Op, DAG);
6309 bool ProduceNonFlagSettingCondBr =
6320 if (!RHS.getNode()) {
6347 if (LHS.getValueType().isInteger()) {
6348 assert((LHS.getValueType() == RHS.getValueType()) &&
6354 if (
RHSC &&
RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
6364 uint64_t
Mask = LHS.getConstantOperandVal(1);
6380 uint64_t
Mask = LHS.getConstantOperandVal(1);
6398 LHS.getOpcode() !=
ISD::AND && ProduceNonFlagSettingCondBr) {
6436 EVT VT =
Op.getValueType();
6443 if (
SrcVT.bitsLT(VT))
6445 else if (
SrcVT.bitsGT(VT))
6510 Attribute::NoImplicitFloat))
6526 EVT VT =
Op.getValueType();
6557 "Unexpected type for custom ctpop lowering");
6564 unsigned EltSize = 8;
6579 EVT VT =
Op.getValueType();
6581 useSVEForFixedLengthVectorVT(VT,
true));
6590 if (
Op.getValueType().isVector())
6591 return LowerVSETCC(Op, DAG);
6598 Chain =
Op.getOperand(0);
6605 EVT VT =
Op.getValueType();
6616 if (!RHS.getNode()) {
6617 assert(LHS.getValueType() ==
Op.getValueType() &&
6618 "Unexpected setcc expansion!");
6623 if (LHS.getValueType().isInteger()) {
6687 if (!RHS.getNode()) {
6700 if (LHS.getValueType().isInteger()) {
6701 assert((LHS.getValueType() == RHS.getValueType()) &&
6743 if (TrueVal == ~FalseVal) {
6745 }
else if (FalseVal > std::numeric_limits<int64_t>::min() &&
6746 TrueVal == -FalseVal) {
6765 }
else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
6768 if (TrueVal > FalseVal) {
6823 assert(LHS.getValueType() == RHS.getValueType());
6841 CTVal &&
CTVal->isZero() &&
TVal.getValueType() == LHS.getValueType())
6845 FVal.getValueType() == LHS.getValueType())
6873 return LowerSELECT_CC(CC, LHS, RHS,
TVal,
FVal,
DL, DAG);
6883 EVT Ty =
Op.getValueType();
6884 if (Ty.isScalableVector()) {
6919 return LowerSELECT_CC(CC, LHS, RHS,
TVal,
FVal,
DL, DAG);
6930 return getAddrLarge(JT, DAG);
6932 return getAddrTiny(JT, DAG);
6934 return getAddr(JT, DAG);
6963 return getGOT(CP, DAG);
6965 return getAddrLarge(CP, DAG);
6967 return getAddrTiny(CP, DAG);
6969 return getAddr(CP, DAG);
6978 return getAddrLarge(BA, DAG);
6980 return getAddrTiny(BA, DAG);
6982 return getAddr(BA, DAG);
7007 : FuncInfo->getVarArgsStackIndex(),
7098 return LowerWin64_VASTART(Op, DAG);
7100 return LowerDarwin_VASTART(Op, DAG);
7102 return LowerAAPCS_VASTART(Op, DAG);
7114 : Subtarget->isTargetILP32() ? 20 : 32;
7126 "automatic va_arg instruction only works on Darwin");
7129 EVT VT =
Op.getValueType();
7139 Chain =
VAList.getValue(1);
7144 "currently not supported");
7198 EVT VT =
Op.getValueType();
7224#define GET_REGISTER_MATCHER
7225#include "AArch64GenAsmMatcher.inc"
7232 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
7234 unsigned DwarfRegNum =
MRI->getDwarfRegNum(Reg,
false);
7248 EVT VT =
Op.getValueType();
7264 EVT VT =
Op.getValueType();
7277 unsigned Reg = MF.
addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
7301 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
7302 EVT VT =
Op.getValueType();
7358 assert(
Op.getNumOperands() == 3 &&
"Not a double-shift!");
7359 EVT VT =
Op.getValueType();
7413 bool OptForSize)
const {
7440 unsigned Limit = (OptForSize ? 1 : (Subtarget->
hasFuseLiterals() ? 5 : 2));
7445 <<
" imm value: "; Imm.
dump(););
7457 if (ST->hasNEON() &&
7469 return DAG.
getNode(Opcode,
SDLoc(Operand), VT, Operand);
7479 EVT VT =
Op.getValueType();
7486AArch64TargetLowering::getSqrtResultForDenormInput(
SDValue Op,
7578const char *AArch64TargetLowering::LowerXConstraint(
EVT ConstraintVT)
const {
7608 if (Constraint ==
"Upa")
7610 if (Constraint ==
"Upl")
7618AArch64TargetLowering::getConstraintType(
StringRef Constraint)
const {
7619 if (Constraint.
size() == 1) {
7620 switch (Constraint[0]) {
7654AArch64TargetLowering::getSingleConstraintMatchWeight(
7657 Value *CallOperandVal =
info.CallOperandVal;
7660 if (!CallOperandVal)
7685std::pair<unsigned, const TargetRegisterClass *>
7686AArch64TargetLowering::getRegForInlineAsmConstraint(
7688 if (Constraint.
size() == 1) {
7689 switch (Constraint[0]) {
7692 return std::make_pair(0U,
nullptr);
7694 return std::make_pair(0U, &AArch64::GPR64commonRegClass);
7695 return std::make_pair(0U, &AArch64::GPR32commonRegClass);
7701 return std::make_pair(0U, &AArch64::ZPRRegClass);
7702 return std::make_pair(0U,
nullptr);
7706 return std::make_pair(0U, &AArch64::FPR16RegClass);
7708 return std::make_pair(0U, &AArch64::FPR32RegClass);
7710 return std::make_pair(0U, &AArch64::FPR64RegClass);
7712 return std::make_pair(0U, &AArch64::FPR128RegClass);
7721 return std::make_pair(0U, &AArch64::ZPR_4bRegClass);
7723 return std::make_pair(0U, &AArch64::FPR128_loRegClass);
7729 return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
7736 return std::make_pair(0U,
nullptr);
7738 return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
7742 if (
StringRef(
"{cc}").equals_lower(Constraint))
7743 return std::make_pair(
unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
7747 std::pair<unsigned, const TargetRegisterClass *> Res;
7753 if ((
Size == 4 ||
Size == 5) && Constraint[0] ==
'{' &&
7754 tolower(Constraint[1]) ==
'v' && Constraint[
Size - 1] ==
'}') {
7757 if (!
Failed && RegNo >= 0 && RegNo <= 31) {
7762 Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
7763 Res.second = &AArch64::FPR64RegClass;
7765 Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
7766 Res.second = &AArch64::FPR128RegClass;
7772 if (Res.second && !Subtarget->
hasFPARMv8() &&
7773 !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
7774 !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
7775 return std::make_pair(0U,
nullptr);
7782void AArch64TargetLowering::LowerAsmOperandForConstraint(
7783 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
7788 if (Constraint.length() != 1)
7813 GA->getValueType(0));
7838 uint64_t CVal =
C->getZExtValue();
7851 uint64_t
NVal = -
C->getSExtValue();
7853 CVal =
C->getSExtValue();
7884 if ((CVal & 0xFFFF) == CVal)
7886 if ((CVal & 0xFFFF0000ULL) == CVal)
7898 if ((CVal & 0xFFFFULL) == CVal)
7900 if ((CVal & 0xFFFF0000ULL) == CVal)
7902 if ((CVal & 0xFFFF00000000ULL) == CVal)
7904 if ((CVal & 0xFFFF000000000000ULL) == CVal)
7906 uint64_t
NCVal = ~CVal;
7913 if ((
NCVal & 0xFFFF000000000000ULL) ==
NCVal)
7927 Ops.push_back(Result);
7954 EVT EltType = V.getValueType().getVectorElementType();
7955 return EltType.getSizeInBits() / 8;
7975 LLVM_DEBUG(
dbgs() <<
"AArch64TargetLowering::ReconstructShuffle\n");
7977 EVT VT = Op.getValueType();
7979 "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
7998 : Vec(Vec),
MinElt(std::numeric_limits<unsigned>::max()),
MaxElt(0),
8014 dbgs() <<
"Reshuffle failed: "
8015 "a shuffle can only come from building a vector from "
8016 "various elements of other vectors, provided their "
8017 "indices are constant\n");
8024 if (Source == Sources.end())
8029 Source->MinElt = std::min(Source->MinElt,
EltNo);
8030 Source->MaxElt = std::max(Source->MaxElt,
EltNo);
8033 if (Sources.size() > 2) {
8035 dbgs() <<
"Reshuffle failed: currently only do something sane when at "
8036 "most two source vectors are involved\n");
8043 for (
auto &Source : Sources) {
8044 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
8058 for (
auto &Src : Sources) {
8059 EVT SrcVT = Src.ShuffleVec.getValueType();
8077 DAG.
getUNDEF(Src.ShuffleVec.getValueType()));
8083 dbgs() <<
"Reshuffle failed: result vector too small to extract\n");
8089 dbgs() <<
"Reshuffle failed: span too large for a VEXT to cope\n");
8114 if (!
SrcVT.is64BitVector()) {
8116 dbgs() <<
"Reshuffle failed: don't know how to lower AArch64ISD::EXT "
8117 "for SVE vectors.");
8124 Src.WindowBase = -Src.MinElt;
8131 for (
auto &Src : Sources) {
8132 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
8139 Src.WindowBase *= Src.WindowScale;
8152 if (Entry.isUndef())
8155 auto Src =
find(Sources, Entry.getOperand(0));
8161 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
8178 LLVM_DEBUG(
dbgs() <<
"Reshuffle failed: illegal shuffle mask\n");
8183 for (
unsigned i = 0;
i < Sources.size(); ++
i)
8191 dbgs() <<
"Reshuffle, creating node: "; V.dump(););
8234 "Only possible block sizes for wide DUP are: 16, 32, 64");
8274 "Shuffle with all-undefs must have been caught by previous cases, "
8316 [&](
int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
8347 "Only possible block sizes for REV are: 16, 32, 64");
8379 if ((M[
i] >= 0 && (
unsigned)M[
i] !=
Idx) ||
8424 if ((M[
i] >= 0 && (
unsigned)M[
i] !=
Idx) ||
8439 for (
unsigned j = 0;
j != 2; ++
j) {
8441 for (
unsigned i = 0;
i != Half; ++
i) {
8442 int MIdx = M[
i +
j * Half];
8529 EVT VT = Op.getValueType();
8538 bool SplitV0 =
V0.getValueSizeInBits() == 128;
8548 if (
V1.getValueSizeInBits() == 128) {
8560 unsigned OpNum = (
PFEntry >> 26) & 0x0F;
8583 if (
LHSID == (1 * 9 + 2) * 9 + 3)
8585 assert(
LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 &&
"Illegal OP_COPY!");
8664 SDValue V2 = Op.getOperand(1);
8667 EVT EltVT = Op.getValueType().getVectorElementType();
8671 for (
int Val : ShuffleMask) {
8672 for (
unsigned Byte = 0; Byte <
BytesPerElt; ++Byte) {
8680 if (Op.getValueSizeInBits() == 128) {
8689 if (V2.getNode()->isUndef()) {
8774 Lane += V.getConstantOperandVal(1);
8775 V = V.getOperand(0);
8792 EVT VT =
Op.getValueType();
8805 if (
SVN->isSplat()) {
8806 int Lane =
SVN->getSplatIndex();
8821 unsigned Opcode =
getDUPLANEOp(
V1.getValueType().getVectorElementType());
8826 for (
unsigned LaneSize : {64U, 32U, 16U}) {
8868 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
8872 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
8876 return DAG.
getNode(Opc, dl,
V1.getValueType(),
V1, V2);
8926 for (
unsigned i = 0;
i != 4; ++
i) {
8927 if (ShuffleMask[
i] < 0)
8937 unsigned Cost = (
PFEntry >> 30);
8949 EVT VT =
Op.getValueType();
8953 if (useSVEForFixedLengthVectorVT(VT))
8954 return LowerToScalableOp(Op, DAG);
8958 switch (
ElemVT.getSimpleVT().SimpleTy) {
8963 if (ConstVal->isOne())
8964 return getPTrue(DAG, dl, VT, AArch64SVEPredPattern::all);
9002 EVT VT =
Op.getValueType();
9016 if (
CIdx && (
CIdx->getZExtValue() <= 3)) {
9049 EVT VT =
BVN->getValueType(0);
9050 APInt SplatBits, SplatUndef;
9051 unsigned SplatBitSize;
9053 if (
BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
HasAnyUndefs)) {
9071 const APInt &Bits) {
9072 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9073 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9074 EVT VT = Op.getValueType();
9093 const SDValue *LHS =
nullptr) {
9094 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9095 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9096 EVT VT = Op.getValueType();
9141 const SDValue *LHS =
nullptr) {
9142 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9143 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9144 EVT VT = Op.getValueType();
9181 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9182 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9183 EVT VT = Op.getValueType();
9211 const APInt &Bits) {
9212 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9213 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9214 EVT VT = Op.getValueType();
9232 const APInt &Bits) {
9233 if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
9234 uint64_t
Value = Bits.zextOrTrunc(64).getZExtValue();
9235 EVT VT = Op.getValueType();
9265 uint64_t &ConstVal) {
9272 EVT VT =
Bvec->getValueType(0);
9277 ConstVal =
FirstElt->getZExtValue();
9282 unsigned Opcode =
N->getOpcode();
9288 if (IID < Intrinsic::num_intrinsics)
9302 EVT VT =
N->getValueType(0);
9357 uint64_t
C2 =
C2node->getZExtValue();
9385 if (useSVEForFixedLengthVectorVT(
Op.getValueType()))
9386 return LowerToScalableOp(Op, DAG);
9392 EVT VT =
Op.getValueType();
9399 LHS =
Op.getOperand(1);
9433 EVT VT = Op.getValueType();
9440 for (
SDValue Lane : Op->ops()) {
9449 }
else if (Lane.getNode()->isUndef()) {
9453 "Unexpected BUILD_VECTOR operand type");
9455 Ops.push_back(Lane);
9461 EVT VT = Op.getValueType();
9503 EVT VT =
Op.getValueType();
9513 if (
BVN->isConstant())
9517 Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
9518 if (Val.isNullValue() || Val.isAllOnesValue())
9567 else if (ConstantValue != V)
9571 if (!
Value.getNode())
9573 else if (V !=
Value) {
9579 if (!
Value.getNode()) {
9581 dbgs() <<
"LowerBUILD_VECTOR: value undefined, creating undef node\n");
9589 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: only low element used, creating 1 "
9590 "SCALAR_TO_VECTOR node\n");
9602 const SDNode *
N = V.getNode();
9628 if (Val - 1 == 2 *
i) {
9660 Value.getValueType() != VT) {
9662 dbgs() <<
"LowerBUILD_VECTOR: use DUP for non-constant splats\n");
9670 if (
Value.getValueSizeInBits() == 64) {
9672 dbgs() <<
"LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
9685 EltTy ==
MVT::f64) &&
"Unsupported floating-point vector type");
9687 dbgs() <<
"LowerBUILD_VECTOR: float constant splats, creating int "
9688 "BITCASTS, and try again\n");
9694 LLVM_DEBUG(
dbgs() <<
"LowerBUILD_VECTOR: trying to lower new vector: ";
9696 Val = LowerBUILD_VECTOR(Val, DAG);
9740 dbgs() <<
"LowerBUILD_VECTOR: all elements are constant, use default "
9755 Ops.push_back(
Value);
9775 dbgs() <<
"LowerBUILD_VECTOR: alternatives failed, creating sequence "
9776 "of INSERT_VECTOR_ELT\n");
9793 LLVM_DEBUG(
dbgs() <<
"Creating node for op0, it is not undefined:\n");
9798 <<
"Creating nodes for the other vector elements:\n";);
9810 dbgs() <<
"LowerBUILD_VECTOR: use default expansion, failed to find "
9811 "better alternative\n");
9817 assert(
Op.getValueType().isScalableVector() &&
9819 "Expected legal scalable vector type!");
9821 if (
isTypeLegal(
Op.getOperand(0).getValueType()) &&
Op.getNumOperands() == 2)
9832 EVT VT =
Op.getOperand(0).getValueType();
9856 Op.getOperand(1),
Op.getOperand(2));
9862AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(
SDValue Op,
9867 EVT VT =
Op.getOperand(0).getValueType();
9901 assert(
Op.getValueType().isFixedLengthVector() &&
9902 "Only cases that extract a fixed length vector are supported!");
9904 EVT InVT =
Op.getOperand(0).getValueType();
9906 unsigned Size =
Op.getValueSizeInBits();
9908 if (
InVT.isScalableVector()) {
9917 if (
Idx == 0 &&
InVT.getSizeInBits() <= 128)
9922 if (
Size == 64 &&
Idx *
InVT.getScalarSizeInBits() == 64 &&
9923 InVT.getSizeInBits() == 128)
9931 assert(
Op.getValueType().isScalableVector() &&
9932 "Only expect to lower inserts into scalable vectors!");
9934 EVT InVT =
Op.getOperand(1).getValueType();
9937 if (
InVT.isScalableVector()) {
9939 EVT VT =
Op.getValueType();
9958 }
else if (
Idx ==
InVT.getVectorMinNumElements()) {
9974 EVT VT =
Op.getValueType();
9976 if (useSVEForFixedLengthVectorVT(VT,
true))
9977 return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
9985 return LowerToPredicatedOp(Op, DAG,
PredOpcode);
10011 if (useSVEForFixedLengthVectorVT(VT))
10017 for (
unsigned i = 0;
i != 4; ++
i) {
10028 unsigned Cost = (
PFEntry >> 30);
10057 Op = Op.getOperand(0);
10059 APInt SplatBits, SplatUndef;
10060 unsigned SplatBitSize;
10062 if (!
BVN || !
BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10075 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10086 assert(VT.
isVector() &&
"vector shift count is not a vector type");
10095 EVT VT =
Op.getValueType();
10100 EVT OpVT =
Op.getOperand(0).getValueType();
10110 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10111 return LowerFixedLengthVectorTruncateToSVE(Op, DAG);
10116SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(
SDValue Op,
10118 EVT VT =
Op.getValueType();
10122 if (!
Op.getOperand(1).getValueType().isVector())
10126 switch (
Op.getOpcode()) {
10134 if (
isVShiftLImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize)
10140 Op.getOperand(0),
Op.getOperand(1));
10146 return LowerToPredicatedOp(Op, DAG, Opc);
10150 if (
isVShiftRImm(
Op.getOperand(1), VT,
false, Cnt) && Cnt < EltSize) {
10153 return DAG.
getNode(Opc,
DL, VT,
Op.getOperand(0),
10160 unsigned Opc = (
Op.getOpcode() ==
ISD::SRA) ? Intrinsic::aarch64_neon_sshl
10179 "function only supposed to emit natural comparisons");
10187 if (
SrcVT.getVectorElementType().isFloatingPoint()) {
10271 if (
Op.getValueType().isScalableVector()) {
10272 if (
Op.getOperand(0).getValueType().isFloatingPoint())
10277 if (useSVEForFixedLengthVectorVT(
Op.getOperand(0).getValueType()))
10278 return LowerFixedLengthVectorSetccToSVE(Op, DAG);
10283 EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
10286 if (LHS.getValueType().getVectorElementType().isInteger()) {
10287 assert(LHS.getValueType() == RHS.getValueType());
10300 if (LHS.getValueType().getVectorNumElements() == 4) {
10311 LHS.getValueType().getVectorElementType() !=
MVT::f128);
10322 if (!
Cmp.getNode())
10328 if (!
Cmp2.getNode())
10362 if (
SrcVT.isScalableVector() ||
10366 return LowerPredReductionToSVE(Op, DAG);
10368 switch (
Op.getOpcode()) {
10398 switch (
Op.getOpcode()) {
10434 MVT VT =
Op.getSimpleValueType();
10439 Op.getOperand(0),
Op.getOperand(1), RHS,
10440 AN->getMemOperand());
10451 MVT VT =
Op.getSimpleValueType();
10456 Op.getOperand(0),
Op.getOperand(1), RHS,
10457 AN->getMemOperand());
10460SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
10489AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(
SDValue Op,
10492 "Only Windows alloca probing supported");
10500 EVT VT =
Node->getValueType(0);
10503 "no-stack-arg-probe")) {
10511 SDValue Ops[2] = {SP, Chain};
10517 Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain,
Size, DAG);
10530 SDValue Ops[2] = {SP, Chain};
10536 EVT VT =
Op.getValueType();
10546template <
unsigned NumVecs>
10565 Info.align.reset();
10576 unsigned Intrinsic)
const {
10577 auto &
DL =
I.getModule()->getDataLayout();
10578 switch (Intrinsic) {
10579 case Intrinsic::aarch64_sve_st2:
10581 case Intrinsic::aarch64_sve_st3:
10583 case Intrinsic::aarch64_sve_st4:
10585 case Intrinsic::aarch64_neon_ld2:
10586 case Intrinsic::aarch64_neon_ld3:
10587 case Intrinsic::aarch64_neon_ld4:
10588 case Intrinsic::aarch64_neon_ld1x2:
10589 case Intrinsic::aarch64_neon_ld1x3:
10590 case Intrinsic::aarch64_neon_ld1x4:
10591 case Intrinsic::aarch64_neon_ld2lane:
10592 case Intrinsic::aarch64_neon_ld3lane:
10593 case Intrinsic::aarch64_neon_ld4lane:
10594 case Intrinsic::aarch64_neon_ld2r:
10595 case Intrinsic::aarch64_neon_ld3r:
10596 case Intrinsic::aarch64_neon_ld4r: {
10599 uint64_t
NumElts =
DL.getTypeSizeInBits(
I.getType()) / 64;
10601 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
10603 Info.align.reset();
10608 case Intrinsic::aarch64_neon_st2:
10609 case Intrinsic::aarch64_neon_st3:
10610 case Intrinsic::aarch64_neon_st4:
10611 case Intrinsic::aarch64_neon_st1x2:
10612 case Intrinsic::aarch64_neon_st1x3:
10613 case Intrinsic::aarch64_neon_st1x4:
10614 case Intrinsic::aarch64_neon_st2lane:
10615 case Intrinsic::aarch64_neon_st3lane:
10616 case Intrinsic::aarch64_neon_st4lane: {
10622 if (!
ArgTy->isVectorTy())
10627 Info.ptrVal =
I.getArgOperand(
I.getNumArgOperands() - 1);
10629 Info.align.reset();
10634 case Intrinsic::aarch64_ldaxr:
10635 case Intrinsic::aarch64_ldxr: {
10639 Info.ptrVal =
I.getArgOperand(0);
10645 case Intrinsic::aarch64_stlxr:
10646 case Intrinsic::aarch64_stxr: {
10650 Info.ptrVal =
I.getArgOperand(1);
10656 case Intrinsic::aarch64_ldaxp:
10657 case Intrinsic::aarch64_ldxp:
10660 Info.ptrVal =
I.getArgOperand(0);
10662 Info.align =
Align(16);
10665 case Intrinsic::aarch64_stlxp:
10666 case Intrinsic::aarch64_stxp:
10669 Info.ptrVal =
I.getArgOperand(2);
10671 Info.align =
Align(16);
10674 case Intrinsic::aarch64_sve_ldnt1: {
10678 Info.ptrVal =
I.getArgOperand(1);
10682 if (Intrinsic == Intrinsic::aarch64_sve_ldnt1)
10686 case Intrinsic::aarch64_sve_stnt1: {
10689 Info.memVT =
MVT::getVT(
I.getOperand(0)->getType());
10690 Info.ptrVal =
I.getArgOperand(2);
10694 if (Intrinsic == Intrinsic::aarch64_sve_stnt1)
10738 if (!
Ty1->isIntegerTy() || !
Ty2->isIntegerTy())
10740 uint64_t
NumBits1 =
Ty1->getPrimitiveSizeInBits().getFixedSize();
10741 uint64_t
NumBits2 =
Ty2->getPrimitiveSizeInBits().getFixedSize();
10745 if (
VT1.isVector() ||
VT2.isVector() || !
VT1.isInteger() || !
VT2.isInteger())
10756 if (
I->getOpcode() != Instruction::FMul)
10759 if (!
I->hasOneUse())
10765 !(
User->getOpcode() == Instruction::FSub ||
10766 User->getOpcode() == Instruction::FAdd))
10783 if (!
Ty1->isIntegerTy() || !
Ty2->isIntegerTy())
10785 unsigned NumBits1 =
Ty1->getPrimitiveSizeInBits();
10786 unsigned NumBits2 =
Ty2->getPrimitiveSizeInBits();
10790 if (
VT1.isVector() ||
VT2.isVector() || !
VT1.isInteger() || !
VT2.isInteger())
10807 return (
VT1.isSimple() && !
VT1.isVector() &&
VT1.isInteger() &&
10808 VT2.isSimple() && !
VT2.isVector() &&
VT2.isInteger() &&
10809 VT1.getSizeInBits() <= 32);
10812bool AArch64TargetLowering::isExtFreeImpl(
const Instruction *Ext)
const {
10817 if (Ext->getType()->isVectorTy())
10820 for (
const Use &U : Ext->
uses()) {
10829 case Instruction::Shl:
10835 auto &
DL = Ext->getModule()->getDataLayout();
10836 std::advance(
GTI, U.getOperandNo()-1);
10842 uint64_t ShiftAmt =
10846 if (ShiftAmt == 0 || ShiftAmt > 4)
10850 case Instruction::Trunc:
10853 if (Instr->
getType() == Ext->getOperand(0)->getType())
10871 auto *HalfTy =
HalfV->getType();
10872 return FullTy->getPrimitiveSizeInBits().getFixedSize() ==
10873 2 * HalfTy->getPrimitiveSizeInBits().getFixedSize();
10879 return FullVT->getNumElements() == 2 *
HalfVT->getNumElements();
10911 return Ext->getType()->getScalarSizeInBits() ==
10912 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
10945 if (!
I->getType()->isVectorTy())
10949 switch (II->getIntrinsicID()) {
10950 case Intrinsic::aarch64_neon_umull:
10953 Ops.push_back(&II->getOperandUse(0));
10954 Ops.push_back(&II->getOperandUse(1));
10957 case Intrinsic::aarch64_neon_pmull64:
10959 II->getArgOperand(1)))
10961 Ops.push_back(&II->getArgOperandUse(0));
10962 Ops.push_back(&II->getArgOperandUse(1));
10970 switch (
I->getOpcode()) {
10971 case Instruction::Sub:
10972 case Instruction::Add: {
10981 Ops.push_back(&
Ext1->getOperandUse(0));
10982 Ops.push_back(&
Ext2->getOperandUse(0));
10985 Ops.push_back(&
I->getOperandUse(0));
10986 Ops.push_back(&
I->getOperandUse(1));
10990 case Instruction::Mul: {
10992 for (
auto &Op :
I->operands()) {
10994 if (
any_of(Ops, [&](
Use *U) {
return U->get() == Op; }))
11017 if (Opcode != Instruction::SExt && Opcode != Instruction::ZExt)
11021 Ops.push_back(&Op);
11040 unsigned NumBits =
LoadedType.getSizeInBits();
11041 return NumBits == 32 || NumBits == 64;
11049 return (
DL.getTypeSizeInBits(VecTy) + 127) / 128;
11063 unsigned VecSize =
DL.getTypeSizeInBits(VecTy);
11094 "Invalid interleave factor");
11095 assert(!Shuffles.empty() &&
"Empty shufflevector input");
11096 assert(Shuffles.size() == Indices.size() &&
11097 "Unmatched number of shufflevectors and indices");
11115 Type *EltTy =
FVTy->getElementType();
11125 if (NumLoads > 1) {
11129 FVTy->getNumElements() / NumLoads);
11134 BaseAddr = Builder.CreateBitCast(
11142 Intrinsic::aarch64_neon_ld3,
11143 Intrinsic::aarch64_neon_ld4};
11157 BaseAddr = Builder.CreateConstGEP1_32(
FVTy->getElementType(), BaseAddr,
11158 FVTy->getNumElements() * Factor);
11161 LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy),
"ldN");
11164 for (
unsigned i = 0;
i < Shuffles.size();
i++) {
11166 unsigned Index = Indices[
i];
11172 SubVec = Builder.CreateIntToPtr(
11174 FVTy->getNumElements()));
11187 SVI->replaceAllUsesWith(
WideVec);
11221 unsigned Factor)
const {
11223 "Invalid interleave factor");
11226 assert(VecTy->getNumElements() % Factor == 0 &&
"Invalid interleaved store");
11228 unsigned LaneLen = VecTy->getNumElements() / Factor;
11229 Type *EltTy = VecTy->getElementType();
11249 Type *IntTy =
DL.getIntPtrType(EltTy);
11255 Op0 = Builder.CreatePtrToInt(Op0,
IntVecTy);
11256 Op1 = Builder.CreatePtrToInt(Op1,
IntVecTy);
11262 Value *BaseAddr =
SI->getPointerOperand();
11264 if (NumStores > 1) {
11273 BaseAddr = Builder.CreateBitCast(
11275 SubVecTy->getElementType()->getPointerTo(
SI->getPointerAddressSpace()));
11280 Type *PtrTy =
SubVecTy->getPointerTo(
SI->getPointerAddressSpace());
11283 Intrinsic::aarch64_neon_st3,
11284 Intrinsic::aarch64_neon_st4};
11288 for (
unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
11293 for (
unsigned i = 0;
i < Factor;
i++) {
11295 if (Mask[
IdxI] >= 0) {
11296 Ops.push_back(Builder.CreateShuffleVector(
11302 if (Mask[
IdxJ * Factor +
IdxI] >= 0) {
11312 Ops.push_back(Builder.CreateShuffleVector(
11319 if (StoreCount > 0)
11320 BaseAddr = Builder.CreateConstGEP1_32(
SubVecTy->getElementType(),
11323 Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
11324 Builder.CreateCall(
StNFunc, Ops);
11349SDValue AArch64TargetLowering::LowerSVEStructLoad(
unsigned Intrinsic,
11355 unsigned N, Opcode;
11356 static std::map<unsigned, std::pair<unsigned, unsigned>>
IntrinsicMap = {
11363 "invalid tuple vector type!");
11376 for (
unsigned I = 0;
I <
N; ++
I)
11445 if (
Immed == std::numeric_limits<int64_t>::min()) {
11447 <<
": avoid UB for INT64_MIN\n");
11453 ((
Immed & 0xfff) == 0 &&
Immed >> 24 == 0));
11455 <<
" legal add imm: " << (
IsLegal ?
"yes" :
"no") <<
"\n");
11491 uint64_t NumBytes = 0;
11492 if (Ty->isSized()) {
11493 uint64_t NumBits =
DL.getTypeSizeInBits(Ty);
11494 NumBytes = NumBits / 8;
11507 unsigned shift =
Log2_64(NumBytes);
11508 if (NumBytes &&
Offset > 0 && (
Offset / NumBytes) <= (1LL << 12) - 1 &&
11517 return AM.
Scale == 1 || (AM.
Scale > 0 && (uint64_t)AM.
Scale == NumBytes);
11527 unsigned AS)
const {
11562 switch (Ty->getScalarType()->getTypeID()) {
11577 AArch64::X16, AArch64::X17, AArch64::LR, 0
11585 N =
N->getOperand(0).getNode();
11586 EVT VT =
N->getValueType(0);
11591 uint64_t
TruncMask =
N->getConstantOperandVal(1);
11593 N->getOperand(0).getOpcode() ==
ISD::SRL &&
11602 assert(Ty->isIntegerTy());
11604 unsigned BitSize = Ty->getPrimitiveSizeInBits();
11612 if ((int64_t)Val < 0)
11615 Val &= (1LL << 32) - 1;
11618 unsigned Shift = (63 - LZ) / 16;
11624 unsigned Index)
const {
11628 return (Index == 0 || Index ==
ResVT.getVectorNumElements());
11637 EVT VT =
N->getValueType(0);
11652 if (!ShiftAmt || ShiftAmt->getZExtValue() !=
ShiftEltTy.getSizeInBits() - 1)
11663 if (!ST->hasDotProd() ||
N->getValueType(0) !=
MVT::i32)
11681 ? Intrinsic::aarch64_neon_udot
11682 : Intrinsic::aarch64_neon_sdot;
11701 Op0 =
AbsOp1.getOperand(0);
11702 Op1 =
AbsOp1.getOperand(1);
11734 if (
DCI.isBeforeLegalizeOps())
11741AArch64TargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
11749 EVT VT =
N->getValueType(0);
11751 !(Divisor.
isPowerOf2() || (-Divisor).isPowerOf2()))
11787 case Intrinsic::aarch64_sve_cntb:
11788 case Intrinsic::aarch64_sve_cnth:
11789 case Intrinsic::aarch64_sve_cntw:
11790 case Intrinsic::aarch64_sve_cntd:
11809 switch (
Extend.getOpcode()) {
11812 return Extend.getOperand(0).getValueType();
11819 return TypeNode->
getVT();
11871 unsigned ExtendOpcode =
Extend.getOpcode();
11924 if (!
Mul->getValueType(0).isVector())
11936 Op0 ? Op0 :
Mul->getOperand(0),
11947 if (
DCI.isBeforeLegalizeOps())
11987 if (
N->hasOneUse() && (
N->use_begin()->getOpcode() ==
ISD::ADD ||
11988 N->use_begin()->getOpcode() ==
ISD::SUB))
11995 unsigned ShiftAmt, AddSubOpc;
12010 }
else if (
CVPlus1.isPowerOf2()) {
12011 ShiftAmt =
CVPlus1.logBase2();
12033 EVT VT =
N->getValueType(0);
12041 "NegateResult and TrailingZeroes cannot both be true for now.");
12065 EVT VT =
N->getValueType(0);
12067 N->getOperand(0)->getOperand(0)->getOpcode() !=
ISD::SETCC ||
12068 VT.
getSizeInBits() !=
N->getOperand(0)->getValueType(0).getSizeInBits())
12078 if (!
BV->isConstant())
12090 N->getOperand(0)->getOperand(0),
MaskConst);
12105 EVT VT =
N->getValueType(0);
12110 if (VT.
getSizeInBits() !=
N->getOperand(0).getValueSizeInBits())
12122 LN0->getPointerInfo(),
LN0->getAlignment(),
12123 LN0->getMemOperand()->getFlags());
12145 if (!
N->getValueType(0).isSimple())
12149 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12157 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
12163 uint32_t IntBits = IntTy.getSizeInBits();
12164 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12173 int32_t Bits = IntBits == 64 ? 64 : 32;
12174 int32_t
C =
BV->getConstantFPSplatPow2ToLog2Int(&
UndefElements, Bits + 1);
12175 if (
C == -1 ||
C == 0 ||
C > Bits)
12179 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12195 "Illegal vector type after legalization");
12199 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
12200 : Intrinsic::aarch64_neon_vcvtfp2fxu;
12221 unsigned Opc = Op->getOpcode();
12222 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
12223 !Op.getOperand(0).getValueType().isSimple() ||
12231 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
12232 int32_t IntBits = IntTy.getSizeInBits();
12233 if (IntBits != 16 && IntBits != 32 && IntBits != 64)
12252 unsigned NumLanes = Op.getValueType().getVectorNumElements();
12274 unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
12275 : Intrinsic::aarch64_neon_vcvtfxu2fp;
12295 ShiftAmount =
N->getConstantOperandVal(1);
12309 EVT VT =
N->getValueType(0);
12347 EVT VT =
N->getValueType(0);
12365 uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
12366 for (
int i = 1;
i >= 0; --
i)
12367 for (
int j = 1;
j >= 0; --
j) {
12378 CN0->getZExtValue() != (BitMask & ~
CN1->getZExtValue())) {
12396 EVT VT =
N->getValueType(0);
12411 if (!
MemVT.getVectorElementType().isSimple())
12415 switch (
MemVT.getVectorElementType().getSimpleVT().SimpleTy) {
12432 return Op0->getAPIntValue().getLimitedValue() ==
MaskForTy;
12439 if (
DCI.isBeforeLegalizeOps())
12456 uint64_t
ExtVal =
C->getZExtValue();
12460 EVT EltTy =
UnpkOp->getValueType(0).getVectorElementType();
12472 UnpkOp->getValueType(0),
12530 EVT VT =
N->getValueType(0);
12577 EVT VT =
N->getValueType(0);
12590 uint64_t ShiftAmt =
C->getZExtValue();
12591 if (VT ==
MVT::i32 && ShiftAmt == 16 &&
12594 if (VT ==
MVT::i64 && ShiftAmt == 32 &&
12616 EVT VT =
N->getValueType(0);
12629 if (ShiftAmount != 1)
12700 EVT VT =
N->getValueType(0);
12726 if (Shuffle && Shuffle->
getMaskElt(0) == 1 &&
12743 EVT VT =
N->getValueType(0);
12764 if (
N00VT ==
N10.getValueType() &&
12769 for (
size_t i = 0;
i < Mask.size(); ++
i)
12781 if (
DCI.isBeforeLegalizeOps())
12795 if (
N->getNumOperands() == 2 &&
N0Opc ==
N1Opc &&
12819 uint64_t
N00Index =
N00.getConstantOperandVal(1);
12820 uint64_t
N01Index =
N01.getConstantOperandVal(1);
12821 uint64_t
N10Index =
N10.getConstantOperandVal(1);
12822 uint64_t
N11Index =
N11.getConstantOperandVal(1);
12852 MVT RHSTy = RHS.getValueType().getSimpleVT();
12854 if (!
RHSTy.isVector())
12858 dbgs() <<
"aarch64-lower: concat_vectors bitcast simplification\n");
12861 RHSTy.getVectorNumElements() * 2);
12873 if (
DCI.isBeforeLegalizeOps())
12900 "unexpected vector size on extract_vector_elt!");
12931 switch (
N.getOpcode()) {
12967 N =
N.getOperand(0);
12971 N.getOperand(0).getValueType().getVectorNumElements() / 2;
13054 isSetCC(Op->getOperand(0), Info));
13064 assert(Op && Op->getOpcode() ==
ISD::ADD &&
"Unexpected operation!");
13065 SDValue LHS = Op->getOperand(0);
13066 SDValue RHS = Op->getOperand(1);
13078 ?
InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
13079 :
InfoAndKind.Info.Generic.Opnd0->getValueType();
13097 EVT VT = Op->getValueType(0);
13104 EVT VT =
N->getValueType(0);
13126 OpVT1.getVectorElementType() != VT)
13131 EVT ValVT =
Val1->getValueType(0);
13153 if (
DCI.isBeforeLegalizeOps())
13156 MVT VT =
N->getSimpleValueType(0);
13168 LHS.getOpcode() != RHS.getOpcode())
13171 unsigned ExtType = LHS.getOpcode();
13177 if (!RHS.getNode())
13183 if (!LHS.getNode())
13212 if (
DCI.isBeforeLegalizeOps())
13217 assert(LHS.getValueType().is64BitVector() &&
13218 RHS.getValueType().is64BitVector() &&
13219 "unexpected shape for long operation");
13226 if (!RHS.getNode())
13230 if (!LHS.getNode())
13235 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N),
N->getValueType(0), LHS, RHS);
13238 N->getOperand(0), LHS, RHS);
13243 unsigned ElemBits = ElemTy.getSizeInBits();
13245 int64_t ShiftAmount;
13248 unsigned SplatBitSize;
13250 if (!
BVN->isConstantSplat(
SplatValue, SplatUndef, SplatBitSize,
13257 ShiftAmount =
CVN->getSExtValue();
13266 case Intrinsic::aarch64_neon_sqshl:
13270 case Intrinsic::aarch64_neon_uqshl:
13274 case Intrinsic::aarch64_neon_srshl:
13278 case Intrinsic::aarch64_neon_urshl:
13282 case Intrinsic::aarch64_neon_sqshlu:
13286 case Intrinsic::aarch64_neon_sshl:
13287 case Intrinsic::aarch64_neon_ushl:
13298 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
13302 return DAG.
getNode(Opcode, dl,
N->getValueType(0),
N->getOperand(1),
13318 if (!
CMask ||
CMask->getZExtValue() != Mask)
13322 N->getOperand(0),
N->getOperand(1),
AndN.getOperand(0));
13330 N->getOperand(1).getSimpleValueType(),
13352 SDValue Scalar =
N->getOperand(3);
13353 EVT ScalarTy = Scalar.getValueType();
13367 EVT VT =
N->getValueType(0);
13393 if (
DCI.isBeforeLegalize())
13400 EVT VT =
N->getValueType(0);
13401 EVT CmpVT =
N->getOperand(2).getValueType();
13412 case Intrinsic::aarch64_sve_cmpeq_wide:
13413 case Intrinsic::aarch64_sve_cmpne_wide:
13414 case Intrinsic::aarch64_sve_cmpge_wide:
13415 case Intrinsic::aarch64_sve_cmpgt_wide:
13416 case Intrinsic::aarch64_sve_cmplt_wide:
13417 case Intrinsic::aarch64_sve_cmple_wide: {
13419 int64_t ImmVal =
CN->getSExtValue();
13420 if (ImmVal >= -16 && ImmVal <= 15)
13428 case Intrinsic::aarch64_sve_cmphs_wide:
13429 case Intrinsic::aarch64_sve_cmphi_wide:
13430 case Intrinsic::aarch64_sve_cmplo_wide:
13431 case Intrinsic::aarch64_sve_cmpls_wide: {
13433 uint64_t ImmVal =
CN->getZExtValue();
13459 assert(Op.getValueType().isScalableVector() &&
13461 "Expected legal scalable vector type!");
13543 assert(
N->getNumOperands() == 4 &&
"Expected 3 operand intrinsic!");
13564 case Intrinsic::aarch64_neon_vcvtfxs2fp:
13565 case Intrinsic::aarch64_neon_vcvtfxu2fp:
13567 case Intrinsic::aarch64_neon_saddv:
13569 case Intrinsic::aarch64_neon_uaddv:
13571 case Intrinsic::aarch64_neon_sminv:
13573 case Intrinsic::aarch64_neon_uminv:
13575 case Intrinsic::aarch64_neon_smaxv:
13577 case Intrinsic::aarch64_neon_umaxv:
13579 case Intrinsic::aarch64_neon_fmax:
13581 N->getOperand(1),
N->getOperand(2));
13582 case Intrinsic::aarch64_neon_fmin:
13584 N->getOperand(1),
N->getOperand(2));
13585 case Intrinsic::aarch64_neon_fmaxnm:
13587 N->getOperand(1),
N->getOperand(2));
13588 case Intrinsic::aarch64_neon_fminnm:
13590 N->getOperand(1),
N->getOperand(2));
13591 case Intrinsic::aarch64_neon_smull:
13592 case Intrinsic::aarch64_neon_umull:
13593 case Intrinsic::aarch64_neon_pmull:
13594 case Intrinsic::aarch64_neon_sqdmull:
13596 case Intrinsic::aarch64_neon_sqshl:
13597 case Intrinsic::aarch64_neon_uqshl:
13598 case Intrinsic::aarch64_neon_sqshlu:
13599 case Intrinsic::aarch64_neon_srshl:
13600 case Intrinsic::aarch64_neon_urshl:
13601 case Intrinsic::aarch64_neon_sshl:
13602 case Intrinsic::aarch64_neon_ushl:
13604 case Intrinsic::aarch64_crc32b:
13605 case Intrinsic::aarch64_crc32cb:
13607 case Intrinsic::aarch64_crc32h:
13608 case Intrinsic::aarch64_crc32ch:
13610 case Intrinsic::aarch64_sve_saddv:
13612 if (
N->getOperand(2)->getValueType(0).getVectorElementType() ==
MVT::i64)
13616 case Intrinsic::aarch64_sve_uaddv:
13618 case Intrinsic::aarch64_sve_smaxv:
13620 case Intrinsic::aarch64_sve_umaxv:
13622 case Intrinsic::aarch64_sve_sminv:
13624 case Intrinsic::aarch64_sve_uminv:
13626 case Intrinsic::aarch64_sve_orv:
13628 case Intrinsic::aarch64_sve_eorv:
13630 case Intrinsic::aarch64_sve_andv:
13632 case Intrinsic::aarch64_sve_index:
13634 case Intrinsic::aarch64_sve_dup:
13636 case Intrinsic::aarch64_sve_dup_x:
13639 case Intrinsic::aarch64_sve_ext:
13641 case Intrinsic::aarch64_sve_smin:
13643 case Intrinsic::aarch64_sve_umin:
13645 case Intrinsic::aarch64_sve_smax:
13647 case Intrinsic::aarch64_sve_umax:
13649 case Intrinsic::aarch64_sve_lsl:
13651 case Intrinsic::aarch64_sve_lsr:
13653 case Intrinsic::aarch64_sve_asr:
13655 case Intrinsic::aarch64_sve_cmphs:
13656 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13658 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13661 case Intrinsic::aarch64_sve_cmphi:
13662 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13664 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13667 case Intrinsic::aarch64_sve_cmpge:
13668 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13670 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13673 case Intrinsic::aarch64_sve_cmpgt:
13674 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13676 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13679 case Intrinsic::aarch64_sve_cmpeq:
13680 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13682 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13685 case Intrinsic::aarch64_sve_cmpne:
13686 if (!
N->getOperand(2).getValueType().isFloatingPoint())
13688 N->getValueType(0),
N->getOperand(1),
N->getOperand(2),
13691 case Intrinsic::aarch64_sve_fadda:
13693 case Intrinsic::aarch64_sve_faddv:
13695 case Intrinsic::aarch64_sve_fmaxnmv:
13697 case Intrinsic::aarch64_sve_fmaxv:
13699 case Intrinsic::aarch64_sve_fminnmv:
13701 case Intrinsic::aarch64_sve_fminv:
13703 case Intrinsic::aarch64_sve_sel:
13705 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
13706 case Intrinsic::aarch64_sve_cmpeq_wide:
13708 case Intrinsic::aarch64_sve_cmpne_wide:
13710 case Intrinsic::aarch64_sve_cmpge_wide:
13712 case Intrinsic::aarch64_sve_cmpgt_wide:
13714 case Intrinsic::aarch64_sve_cmplt_wide:
13716 case Intrinsic::aarch64_sve_cmple_wide:
13718 case Intrinsic::aarch64_sve_cmphs_wide:
13720 case Intrinsic::aarch64_sve_cmphi_wide:
13722 case Intrinsic::aarch64_sve_cmplo_wide:
13724 case Intrinsic::aarch64_sve_cmpls_wide:
13726 case Intrinsic::aarch64_sve_ptest_any:
13729 case Intrinsic::aarch64_sve_ptest_first:
13732 case Intrinsic::aarch64_sve_ptest_last:
13782 if (!
DCI.isBeforeLegalizeOps())
13802 if (
SrcVT.getSizeInBits().getKnownMinSize() != 64)
13818 LoVT.getVectorElementCount());
13833 assert(!
St.isTruncatingStore() &&
"cannot split truncating vector store");
13843 uint64_t BaseOffset = 0;
13851 if (BasePtr->getOpcode() ==
ISD::ADD &&
13854 BasePtr = BasePtr->getOperand(0);
13865 St.getMemOperand()->getFlags());
13874 assert(
ContentTy.isSimple() &&
"No SVE containers for extended types");
13876 switch (
ContentTy.getSimpleVT().SimpleTy) {
13903 EVT VT =
N->getValueType(0);
13929 EVT VT =
N->getValueType(0);
13930 EVT PtrTy =
N->getOperand(3).getValueType();
13956template <
unsigned Opcode>
13960 "Unsupported opcode.");
13962 EVT VT =
N->getValueType(0);
13992 if (
DataVT.isFloatingPoint())
13996 if (
Data.getValueType().isFloatingPoint())
14016 EVT PtrTy =
N->getOperand(4).getValueType();
14022 if (
DataVT.isFloatingPoint())
14070 if (!
StVal.hasOneUse())
14075 if (
St.isTruncatingStore())
14081 int64_t
Offset =
St.getBasePtr()->getConstantOperandVal(1);
14130 if (
St.isTruncatingStore())
14268 if (
DCI.isBeforeLegalizeOps())
14272 EVT VT =
N->getValueType(0);
14302 if (UI.getUse().getResNo() == 1)
14312 Addr.getNode()->use_end(); UI != UE; ++UI) {
14315 || UI.getUse().getResNo() !=
Addr.getResNo())
14332 Visited.insert(
Addr.getNode());
14333 Worklist.push_back(
User);
14334 Worklist.push_back(LD);
14335 Worklist.push_back(
Vector.getNode());
14341 Ops.push_back(LD->getOperand(0));
14344 Ops.push_back(Lane);
14346 Ops.push_back(
Addr);
14347 Ops.push_back(Inc);
14378 !
DCI.isBeforeLegalizeOps());
14381 DCI.CommitTargetLoweringOpt(
TLO);
14405 assert(
MGS &&
"Can only combine gather load or scatter store nodes");
14417 if (
DCI.isBeforeLegalize()) {
14423 if (
MGS->isIndexSigned())
14433 MGT->getMemOperand(),
14434 MGT->getIndexType(),
MGT->getExtensionType());
14440 MSC->getMemoryVT(),
DL, Ops,
14441 MSC->getMemOperand(), IndexType,
14442 MSC->isTruncatingStore());
14455 if (
DCI.isBeforeLegalize() ||
DCI.isCalledByLegalizer())
14458 unsigned AddrOpIdx =
N->getNumOperands() - 1;
14463 UE =
Addr.getNode()->use_end(); UI != UE; ++UI) {
14466 UI.getUse().getResNo() !=
Addr.getResNo())
14473 Visited.insert(
Addr.getNode());
14474 Worklist.push_back(
N);
14475 Worklist.push_back(
User);
14481 bool IsStore =
false;
14496 NumVecs = 2; IsStore =
true;
break;
14498 NumVecs = 3; IsStore =
true;
break;
14500 NumVecs = 4; IsStore =
true;
break;
14508 NumVecs = 2; IsStore =
true;
break;
14510 NumVecs = 3; IsStore =
true;
break;
14512 NumVecs = 4; IsStore =
true;
break;
14535 VecTy =
N->getOperand(2).getValueType();
14537 VecTy =
N->getValueType(0);
14551 Ops.push_back(
N->getOperand(0));
14555 Ops.push_back(
N->getOperand(
i));
14556 Ops.push_back(
Addr);
14557 Ops.push_back(Inc);
14572 MemInt->getMemOperand());
14593 switch(V.getNode()->getOpcode()) {
14600 ExtType =
LoadNode->getExtensionType();
14626 1LL << (width - 1);
14794 else if (
CNV == 65535)
14866 unsigned CmpOpc = Cmp.getOpcode();
14872 if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1))
14875 SDValue LHS = Cmp.getOperand(0);
14876 SDValue RHS = Cmp.getOperand(1);
14878 assert(LHS.getValueType() == RHS.getValueType() &&
14879 "Expected the value type to be the same for both operands!");
14901 DCI.CombineTo(
N, BR,
false);
14913 if (!Op->hasOneUse())
14923 Bit < Op->getValueType(0).getSizeInBits()) {
14929 Bit < Op->getOperand(0).getValueSizeInBits()) {
14933 if (Op->getNumOperands() != 2)
14940 switch (Op->getOpcode()) {
14946 if ((
C->getZExtValue() >> Bit) & 1)
14952 if (
C->getZExtValue() <= Bit &&
14953 (Bit -
C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
14954 Bit = Bit -
C->getZExtValue();
14961 Bit = Bit +
C->getZExtValue();
14962 if (Bit >= Op->getValueType(0).getSizeInBits())
14963 Bit = Op->getValueType(0).getSizeInBits() - 1;
14968 if ((Bit +
C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
14969 Bit = Bit +
C->getZExtValue();
14976 if ((
C->getZExtValue() >> Bit) & 1)
14994 unsigned NewOpc =
N->getOpcode();
15026 if (
ResVT.getSizeInBits() !=
CmpVT.getSizeInBits())
15056 "Scalar-SETCC feeding SELECT has unexpected result type!");
15078 if (
CCVT.getSizeInBits() !=
ResVT.getSizeInBits())
15098 ResVT.changeVectorElementTypeToInteger(), Mask);
15105 if (
N->getValueType(0) ==
N->getOperand(0).getValueType())
15106 return N->getOperand(0);
15122 uint64_t MinOffset = -1ull;
15131 MinOffset = std::min(MinOffset,
C->getZExtValue());
15133 uint64_t
Offset = MinOffset +
GN->getOffset();
15138 if (
Offset <= uint64_t(
GN->getOffset()))
15149 if (
Offset >= (1 << 21))
15154 if (!
T->isSized() ||
15169 "This method is only for scalable vectors of offsets");
15218 "Scatter stores are only possible for SVE vectors");
15228 if (
SrcElVT.isFloatingPoint())
15264 SrcVT.getScalarSizeInBits() / 8)) {
15285 if (!TLI.isTypeLegal(
Offset.getValueType()))
15295 if (
SrcVT.isFloatingPoint())
15313 return DAG.
getNode(Opcode,
DL, VTs, Ops);
15321 "Gather loads are only possible for SVE vectors");
15341 RetVT.getScalarSizeInBits());
15351 Offset.getValueType().isVector())
15363 RetVT.getScalarSizeInBits() / 8)) {
15395 if (
RetVT.isFloatingPoint())
15411 if (
RetVT.isFloatingPoint())
15444 "Sign extending from an invalid type");
15454 if (
DCI.isBeforeLegalizeOps())
15586 Ops[1] = DAG.
getConstant(Intrinsic::aarch64_sve_prfb_gather_uxtw_index,
DL,
15595 switch (
N->getOpcode()) {
15669 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
15671 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
15673 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
15675 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
15677 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
15678 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
15679 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
15680 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
15681 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
15682 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
15683 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
15684 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
15686 case Intrinsic::aarch64_neon_ld2:
15687 case Intrinsic::aarch64_neon_ld3:
15688 case Intrinsic::aarch64_neon_ld4:
15689 case Intrinsic::aarch64_neon_ld1x2:
15690 case Intrinsic::aarch64_neon_ld1x3:
15691 case Intrinsic::aarch64_neon_ld1x4:
15692 case Intrinsic::aarch64_neon_ld2lane:
15693 case Intrinsic::aarch64_neon_ld3lane:
15694 case Intrinsic::aarch64_neon_ld4lane:
15695 case Intrinsic::aarch64_neon_ld2r:
15696 case Intrinsic::aarch64_neon_ld3r:
15697 case Intrinsic::aarch64_neon_ld4r:
15698 case Intrinsic::aarch64_neon_st2:
15699 case Intrinsic::aarch64_neon_st3:
15700 case Intrinsic::aarch64_neon_st4:
15701 case Intrinsic::aarch64_neon_st1x2:
15702 case Intrinsic::aarch64_neon_st1x3:
15703 case Intrinsic::aarch64_neon_st1x4:
15704 case Intrinsic::aarch64_neon_st2lane:
15705 case Intrinsic::aarch64_neon_st3lane:
15706 case Intrinsic::aarch64_neon_st4lane:
15708 case Intrinsic::aarch64_sve_ldnt1:
15710 case Intrinsic::aarch64_sve_ld1rq:
15712 case Intrinsic::aarch64_sve_ld1ro:
15714 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
15716 case Intrinsic::aarch64_sve_ldnt1_gather:
15718 case Intrinsic::aarch64_sve_ldnt1_gather_index:
15721 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
15723 case Intrinsic::aarch64_sve_ld1:
15725 case Intrinsic::aarch64_sve_ldnf1:
15727 case Intrinsic::aarch64_sve_ldff1:
15729 case Intrinsic::aarch64_sve_st1:
15731 case Intrinsic::aarch64_sve_stnt1:
15733 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
15735 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
15737 case Intrinsic::aarch64_sve_stnt1_scatter:
15739 case Intrinsic::aarch64_sve_stnt1_scatter_index:
15741 case Intrinsic::aarch64_sve_ld1_gather:
15743 case Intrinsic::aarch64_sve_ld1_gather_index:
15746 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
15749 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
15752 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
15756 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
15760 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
15762 case Intrinsic::aarch64_sve_ldff1_gather:
15764 case Intrinsic::aarch64_sve_ldff1_gather_index:
15767 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
15771 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
15775 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
15779 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
15783 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
15786 case Intrinsic::aarch64_sve_st1_scatter:
15788 case Intrinsic::aarch64_sve_st1_scatter_index:
15790 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
15793 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
15796 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
15800 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
15804 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
15806 case Intrinsic::aarch64_sve_tuple_get: {
15814 uint64_t
NumLanes =
ResVT.getVectorElementCount().getKnownMinValue();
15820 case Intrinsic::aarch64_sve_tuple_set: {
15842 Opnds.push_back(Vec);
15853 case Intrinsic::aarch64_sve_tuple_create2:
15854 case Intrinsic::aarch64_sve_tuple_create3:
15855 case Intrinsic::aarch64_sve_tuple_create4: {
15860 for (
unsigned I = 2;
I <
N->getNumOperands(); ++
I)
15861 Opnds.push_back(
N->getOperand(
I));
15867 (
N->getNumOperands() - 2));
15871 case Intrinsic::aarch64_sve_ld2:
15872 case Intrinsic::aarch64_sve_ld3:
15873 case Intrinsic::aarch64_sve_ld4: {
15877 SDValue BasePtr =
N->getOperand(3);
15879 unsigned IntrinsicID =
15882 LowerSVEStructLoad(IntrinsicID,
LoadOps,
N->getValueType(0), DAG,
DL);
15899bool AArch64TargetLowering::isUsedByReturnOnly(
SDNode *
N,
15901 if (
N->getNumValues() != 1)
15903 if (!
N->hasNUsesOfValue(1, 0))
15907 SDNode *Copy = *
N->use_begin();
15911 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
15914 TCChain = Copy->getOperand(0);
15936bool AArch64TargetLowering::mayBeEmittedAsTailCall(
const CallInst *CI)
const {
15940bool AArch64TargetLowering::getIndexedAddressParts(
SDNode *Op,
SDValue &Base,
15948 Base =
Op->getOperand(0);
15952 int64_t
RHSC = RHS->getSExtValue();
15964bool AArch64TargetLowering::getPreIndexedAddressParts(
SDNode *
N,
SDValue &Base,
15971 VT =
LD->getMemoryVT();
15972 Ptr =
LD->getBasePtr();
15974 VT =
ST->getMemoryVT();
15975 Ptr =
ST->getBasePtr();
15986bool AArch64TargetLowering::getPostIndexedAddressParts(
15992 VT =
LD->getMemoryVT();
15993 Ptr =
LD->getBasePtr();
15995 VT =
ST->getMemoryVT();
15996 Ptr =
ST->getBasePtr();
16001 if (!getIndexedAddressParts(Op, Base,
Offset, AM,
IsInc, DAG))
16049 return std::make_pair(Lo, Hi);
16052void AArch64TargetLowering::ReplaceExtractSubVectorResults(
16058 if (!
InVT.isScalableVector() || !
InVT.isInteger())
16062 EVT VT =
N->getValueType(0);
16068 if (
InVT.getVectorElementCount() != (
ResEC * 2))
16075 unsigned Index =
CIndex->getZExtValue();
16076 if ((Index != 0) && (Index !=
ResEC.getKnownMinValue()))
16088 SDLoc dl(V.getNode());
16109 "AtomicCmpSwap on types less than 128 should be legal");
16124 switch (
MemOp->getOrdering()) {
16126 Opcode = AArch64::CASPX;
16129 Opcode = AArch64::CASPAX;
16132 Opcode = AArch64::CASPLX;
16136 Opcode = AArch64::CASPALX;
16162 New.first, New.second,
N->getOperand(0)};
16164 AArch64::CMP_SWAP_128,
SDLoc(
N),
16175void AArch64TargetLowering::ReplaceNodeResults(
16177 switch (
N->getOpcode()) {
16215 assert(
N->getValueType(0) ==
MVT::i128 &&
"unexpected illegal conversion");
16223 "unexpected load's value type");
16233 DAG.
getVTList({MVT::i64, MVT::i64, MVT::Other}),
16234 {LoadNode->getChain(), LoadNode->getBasePtr()},
LoadNode->getMemoryVT(),
16243 ReplaceExtractSubVectorResults(
N,
Results, DAG);
16246 EVT VT =
N->getValueType(0);
16248 "custom lowering for unexpected type");
16255 case Intrinsic::aarch64_sve_clasta_n: {
16259 N->getOperand(1), Op2,
N->getOperand(3));
16263 case Intrinsic::aarch64_sve_clastb_n: {
16267 N->getOperand(1), Op2,
N->getOperand(3));
16271 case Intrinsic::aarch64_sve_lasta: {
16274 N->getOperand(1),
N->getOperand(2));
16278 case Intrinsic::aarch64_sve_lastb: {
16281 N->getOperand(1),
N->getOperand(2));
16296unsigned AArch64TargetLowering::combineRepeatedFPDivisors()
const {
16317 unsigned Size =
SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
16318 return Size == 128;
16378 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16385 if (ValTy->getPrimitiveSizeInBits() == 128) {
16387 IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
16393 Value *Lo = Builder.CreateExtractValue(
LoHi, 0,
"lo");
16394 Value *Hi = Builder.CreateExtractValue(
LoHi, 1,
"hi");
16395 Lo = Builder.CreateZExt(Lo, ValTy,
"lo64");
16396 Hi = Builder.CreateZExt(Hi, ValTy,
"hi64");
16397 return Builder.CreateOr(
16403 IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
16412 return Builder.CreateBitCast(Trunc, EltTy);
16417 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16424 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
16432 IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
16436 Value *Lo = Builder.CreateTrunc(Val, Int64Ty,
"lo");
16437 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty,
"hi");
16439 return Builder.CreateCall(
Stxr, {Lo, Hi,
Addr});
16443 IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
16449 Val = Builder.CreateBitCast(Val,
IntValTy);
16451 return Builder.CreateCall(
Stxr,
16452 {Builder.CreateZExtOrBitCast(
16453 Val,
Stxr->getFunctionType()->getParamType(0)),
16459 if (Ty->isArrayTy())
16463 if (
TySize.isScalable() &&
TySize.getKnownMinSize() > 128)
16469bool AArch64TargetLowering::shouldNormalizeToSelectSequence(
LLVMContext &,
16475 Module *M = IRB.GetInsertBlock()->getParent()->getParent();
16478 return IRB.CreatePointerCast(
16481 IRB.getInt8PtrTy()->getPointerTo(0));
16503 M.getOrInsertGlobal(
"__security_cookie",
16512 F->addAttribute(1, Attribute::AttrKind::InReg);
16522 return M.getGlobalVariable(
"__security_cookie");
16558 return Mask->getValue().isPowerOf2();
16601 if (AArch64::GPR64RegClass.
contains(*
I))
16602 RC = &AArch64::GPR64RegClass;
16603 else if (AArch64::FPR64RegClass.
contains(*
I))
16604 RC = &AArch64::FPR64RegClass;
16614 assert(Entry->getParent()->getFunction().hasFnAttribute(
16615 Attribute::NoUnwind) &&
16616 "Function should be nounwind in insertCopiesSplitCSR!");
16617 Entry->addLiveIn(*
I);
16622 for (
auto *Exit : Exits)
16624 TII->get(TargetOpcode::COPY), *
I)
16658void AArch64TargetLowering::finalizeLowering(
MachineFunction &MF)
const {
16668bool AArch64TargetLowering::shouldLocalize(
16670 switch (
MI.getOpcode()) {
16671 case TargetOpcode::G_GLOBAL_VALUE: {
16682 case AArch64::ADRP:
16683 case AArch64::G_ADD_LOW:
16711 "Expected legal fixed length vector!");
16737 "Expected legal fixed length vector!");
16744 PgPattern = AArch64SVEPredPattern::vl1;
16747 PgPattern = AArch64SVEPredPattern::vl2;
16750 PgPattern = AArch64SVEPredPattern::vl4;
16753 PgPattern = AArch64SVEPredPattern::vl8;
16756 PgPattern = AArch64SVEPredPattern::vl16;
16759 PgPattern = AArch64SVEPredPattern::vl32;
16762 PgPattern = AArch64SVEPredPattern::vl64;
16765 PgPattern = AArch64SVEPredPattern::vl128;
16768 PgPattern = AArch64SVEPredPattern::vl256;
16804 "Expected legal scalable vector!");
16819 "Expected to convert into a scalable vector!");
16820 assert(V.getValueType().isFixedLengthVector() &&
16821 "Expected a fixed length vector operand!");
16830 "Expected to convert into a fixed length vector!");
16831 assert(V.getValueType().isScalableVector() &&
16832 "Expected a scalable vector operand!");
16839SDValue AArch64TargetLowering::LowerFixedLengthVectorLoadToSVE(
16844 EVT VT =
Op.getValueType();
16850 Load->getMemoryVT(),
Load->getMemOperand(),
Load->getAddressingMode(),
16851 Load->getExtensionType());
16859SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE(
16864 EVT VT =
Store->getValue().getValueType();
16871 Store->getMemOperand(),
Store->getAddressingMode(),
16872 Store->isTruncatingStore());
16875SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
16878 EVT VT =
Op.getValueType();
16886 return LowerToPredicatedOp(Op, DAG,
PredOpcode,
true);
16925SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
16927 EVT VT =
Op.getValueType();
16961SDValue AArch64TargetLowering::LowerFixedLengthVectorTruncateToSVE(
16963 EVT VT =
Op.getValueType();
17004 EVT VT =
Op.getValueType();
17013 for (
const SDValue &V :
Op->op_values()) {
17027 "Only fixed length vectors are supported!");
17041 for (
const SDValue &V :
Op->op_values()) {
17042 assert((!V.getValueType().isVector() ||
17043 V.getValueType().isScalableVector()) &&
17044 "Only scalable vectors are supported!");
17059 EVT VT =
Op.getValueType();
17060 assert(useSVEForFixedLengthVectorVT(VT) &&
17061 "Only expected to lower fixed length vector operation!");
17066 for (
const SDValue &V :
Op->op_values()) {
17070 if (!V.getValueType().isVector()) {
17076 assert(useSVEForFixedLengthVectorVT(V.getValueType()) &&
17077 "Only fixed length vectors are supported!");
17094 if (
SrcVT.isFixedLengthVector()) {
17120 if (!
OpVT.isScalableVector() ||
OpVT.getVectorElementType() !=
MVT::i1)
17146SDValue AArch64TargetLowering::LowerReductionToSVE(
unsigned Opcode,
17153 if (useSVEForFixedLengthVectorVT(
SrcVT,
true)) {
17160 SrcVT.getVectorElementType();
17178AArch64TargetLowering::LowerFixedLengthVectorSelectToSVE(
SDValue Op,
17180 EVT VT =
Op.getValueType();
17183 EVT InVT =
Op.getOperand(1).getValueType();
17202SDValue AArch64TargetLowering::LowerFixedLengthVectorSetccToSVE(
17205 EVT InVT =
Op.getOperand(0).getValueType();
17208 assert(useSVEForFixedLengthVectorVT(
InVT) &&
17209 "Only expected to lower fixed length vector operation!");
17210 assert(
Op.getValueType() ==
InVT.changeTypeToInteger() &&
17211 "Expected integer result of the same bit length as the inputs!");
17214 if (
InVT.isFloatingPoint())
17223 {Pg, Op1, Op2,
Op.getOperand(2)});
17239 "Only expect to cast between legal scalable vector types!");
17242 "Cannot cast between data and predicate scalable vector types!");
17253 "Cannot cast between unpacked scalable vector types!");
unsigned const MachineRegisterInfo * MRI
if(Register::isVirtualRegister(Reg)) return MRI -> getRegClass(Reg) ->hasSuperClassEq(&AArch64::GPR64RegClass)
static unsigned MatchRegisterName(StringRef Name)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isConcatMask(ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A))...
static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector...
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isSingletonEXTMask(ArrayRef< int > M, EVT VT, unsigned &Imm)
static SDValue performCONDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue NormalizeBuildVector(SDValue Op, SelectionDAG &DAG)
static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
static SDValue GenerateTBL(SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static std::pair< SDValue, SDValue > splitInt128(SDValue N, SelectionDAG &DAG)
static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo)
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
static bool isLegalArithImmed(uint64_t C)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performSTNT1Combine(SDNode *N, SelectionDAG &DAG)
unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static bool areExtractShuffleVectors(Value *Op1, Value *Op2)
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector ...
unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend)
static bool isREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
static SDValue performSRLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point divide by power of two into fixed-point to floating-point conversion.
static SDValue getScaledOffsetForBitWidth(SelectionDAG &DAG, SDValue Offset, SDLoc DL, unsigned BitWidth)
static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG)
static bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performBRCONDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static MVT getSVEContainerType(EVT ContentTy)
static bool isMergePassthruOpcode(unsigned Opc)
static EVT calculatePreExtendType(SDValue Extend, SelectionDAG &DAG)
Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
static SDValue performNEONPostLDSTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
static void ReplaceCMP_SWAP_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static bool areExtractExts(Value *Ext1, Value *Ext2)
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
static SDValue performSelectCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instruct...
static bool canGuaranteeTCO(CallingConv::ID CC)
Return true if the calling convention is one that we can guarantee TCO for.
static SDValue performABSCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static cl::opt< bool > EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))
static bool isUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isValidImmForSVEVecImmAddrMode(unsigned OffsetInBytes, unsigned ScalarSizeInBytes)
Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scat...
static bool isUZP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v,...
static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue convertMergedOpToPredOp(SDNode *N, unsigned PredOpc, SelectionDAG &DAG)
static unsigned getDUPLANEOp(EVT EltType)
static void changeFPCCToAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static bool isTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM)
static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI)
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG)
static SDValue performNVCASTCombine(SDNode *N)
Get rid of unnecessary NVCASTs (that don't change the type).
static EVT getPackedSVEVectorVT(EVT VT)
static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG)
static void ReplaceReductionResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi)
An EXTR instruction is made up of two shifts, ORed together.
static bool isEquivalentMaskless(unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant)
static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG)
static EVT getExtensionTo64Bits(const EVT &OrigVT)
static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT, unsigned Opcode, SelectionDAG &DAG)
static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N, SelectionDAG &DAG)
static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static bool isINSMask(ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG)
static unsigned getIntrinsicID(const SDNode *N)
static bool IsSVECntIntrinsic(SDValue S)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool canEmitConjunction(const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0)
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
static bool isWideDUPMask(ArrayRef< int > M, EVT VT, unsigned BlockSize, unsigned &DupLaneOp)
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vect...
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, bool isSigned)
static SDValue performUzpCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performST1Combine(SDNode *N, SelectionDAG &DAG)
static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *ST)
static SDValue performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineSVEReductionOrderedFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static SDValue legalizeSVEGatherPrefetchOffsVec(SDNode *N, SelectionDAG &DAG)
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked ...
static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG)
static SDValue performLD1Combine(SDNode *N, SelectionDAG &DAG, unsigned Opc)
static bool hasPairwiseAdd(unsigned Opcode, EVT VT, bool FullFP16)
static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG)
static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue combineSVEPrefetchVecBaseImmOff(SDNode *N, SelectionDAG &DAG, unsigned ScalarSizeInBytes)
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses ...
static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St)
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
unsigned getSignExtendedGatherOpcode(unsigned Opcode)
static SDValue performAddSubCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG)
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt)
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift oper...
static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC)
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
void selectGatherScatterAddrMode(SDValue &BasePtr, SDValue &Index, EVT MemVT, unsigned &Opcode, bool IsGather, SelectionDAG &DAG)
static SDValue combineSVEReductionFP(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static unsigned getCmpOperandFoldingProfit(SDValue Op)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static SDValue performConcatVectorsCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG)
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG)
static SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT, int Pattern)
static bool isEXTMask(ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
static SDValue tryCombineFixedPointConvert(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT)
static SDValue performMulVectorExtendCombine(SDNode *Mul, SelectionDAG &DAG)
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZE...
static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorTruncateCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
static EVT getPromotedVTForPredicate(EVT VT)
static void changeFPCCToANDAArch64CC(ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
Convert a DAG fp condition code to an AArch64 CC.
static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X,...
static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG)
static bool isAllConstantBuildVector(const SDValue &PotentialBVec, uint64_t &ConstVal)
static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
static SDValue performLD1ReplicateCombine(SDNode *N, SelectionDAG &DAG)
static SDValue getPTest(SelectionDAG &DAG, EVT VT, SDValue Pg, SDValue Op, AArch64CC::CondCode Cond)
static bool isSetCCOrZExtSetCC(const SDValue &Op, SetCCInfoAndKind &Info)
cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations t...
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG)
static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG)
static SDValue performPostLD1Combine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2)
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
std::pair< SDValue, uint64_t > lookThroughSignExtension(SDValue Val)
static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG)
static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static PredicateConstraint parsePredicateConstraint(StringRef Constraint)
static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
static bool isPackedVectorType(EVT VT, SelectionDAG &DAG)
Returns true if VT's elements occupy the lowest bit positions of its associated register class withou...
static SDValue performAddSubLongCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static bool isTRN_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v,...
static bool isZIPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left...
static SDValue performExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performMaskedGatherScatterCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG)
static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl, SelectionDAG &DAG, SDValue Chain, bool IsSignaling)
static bool isSignExtended(SDNode *N, SelectionDAG &DAG)
static SDValue performUADDVCombine(SDNode *N, SelectionDAG &DAG)
bool getGatherScatterIndexIsExtended(SDValue Index)
static bool isZeroExtended(SDNode *N, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue performScatterStoreCombine(SDNode *N, SelectionDAG &DAG, unsigned Opcode, bool OnlyPackedOffsets=true)
static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue combineSVEReductionInt(SDNode *N, unsigned Opc, SelectionDAG &DAG)
static bool isCMN(SDValue Op, ISD::CondCode CC)
static SDValue tryCombineToEXTR(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low...
static bool isOperandOfVmullHighP64(Value *Op)
Check if Op could be used with vmull_high_p64 intrinsic.
static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static bool isEssentiallyExtractHighSubvector(SDValue N)
static bool mayTailCallThisCC(CallingConv::ID CC)
Return true if we might ever do TCO for calls with this calling convention.
static Value * UseTlsOffset(IRBuilder<> &IRB, unsigned Offset)
static unsigned getExtFactor(SDValue &V)
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vect...
static SDValue performExtractVectorEltCombine(SDNode *N, SelectionDAG &DAG)
static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue performLDNT1Combine(SDNode *N, SelectionDAG &DAG)
static const MVT MVT_CC
Value type used for condition codes.
static SDValue performCommonVectorExtendCombine(SDValue VectorShuffle, SelectionDAG &DAG)
Combines a dup(sext/zext) node pattern into sext/zext(dup) making use of the vector SExt/ZExt rather ...
static SDValue performTBZCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl)
static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
Simplify Addr given that the top byte of it is ignored by HW during address translation.
static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static cl::opt< bool > EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true))
static bool isZIP_v_undef_Mask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v,...
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static std::pair< SDValue, SDValue > getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
#define FALKOR_STRIDED_ACCESS_MD
static const unsigned PerfectShuffleTable[6561+1]
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static bool isConstant(const MachineInstr &MI)
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static const MCPhysReg GPRArgRegs[]
Function Alias Analysis Results
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
PropagateLiveness Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define STATISTIC(VARNAME, DESC)
static const int BlockSize
This defines the Use class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static constexpr int Concat[]
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
unsigned getVarArgsFPRSize() const
void setVarArgsStackIndex(int Index)
SmallVectorImpl< ForwardedRegister > & getForwardedMustTailRegParms()
void setIsSplitCSR(bool s)
int getVarArgsFPRIndex() const
void incNumLocalDynamicTLSAccesses()
void setBytesInStackArgArea(unsigned bytes)
int getVarArgsStackIndex() const
void setVarArgsGPRIndex(int Index)
int getVarArgsGPRIndex() const
void setVarArgsFPRSize(unsigned Size)
unsigned getVarArgsGPRSize() const
unsigned getSRetReturnReg() const
void setSRetReturnReg(unsigned Reg)
unsigned getBytesInStackArgArea() const
void setJumpTableEntryInfo(int Idx, unsigned Size, MCSymbol *PCRelSym)
void setVarArgsFPRIndex(int Index)
void setVarArgsGPRSize(unsigned Size)
void setArgumentStackToRestore(unsigned bytes)
void UpdateCustomCalleeSavedRegs(MachineFunction &MF) const
static bool hasSVEArgsOrReturn(const MachineFunction *MF)
bool isTargetWindows() const
bool hasFuseLiterals() const
unsigned getPrefLoopLogAlignment() const
const AArch64RegisterInfo * getRegisterInfo() const override
unsigned getPrefFunctionLogAlignment() const
bool isMisaligned128StoreSlow() const
const AArch64InstrInfo * getInstrInfo() const override
unsigned getMaximumJumpTableSize() const
bool isTargetDarwin() const
bool isTargetILP32() const
ARMProcFamilyEnum getProcFamily() const
Returns ARM processor family.
unsigned classifyGlobalFunctionReference(const GlobalValue *GV, const TargetMachine &TM) const
bool isTargetMachO() const
bool supportsAddressTopByteIgnored() const
CPU has TBI (top byte of addresses is ignored during HW address translation) and OS enables it.
bool isTargetAndroid() const
const Triple & getTargetTriple() const
bool isCallingConvWin64(CallingConv::ID CC) const
unsigned getMinSVEVectorSizeInBits() const
bool outlineAtomics() const
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
bool isLittleEndian() const
bool hasAggressiveFMA() const
bool isXRegisterReserved(size_t i) const
bool requiresStrictAlign() const
bool isTargetFuchsia() const
bool predictableSelectIsExpensive() const
bool useSVEForFixedLengthVectors() const
bool hasCustomCallingConv() const
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
unsigned getVaListSizeInBits(const DataLayout &DL) const override
Returns the size of the platform's va_list object.
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns true if the given (atomic) store should be expanded by the IR-level AtomicExpand pass into an...
int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS) const override
Return the cost of the scaling factor used in the addressing mode represented by AM for this target,...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
Value * emitLoadLinked(IRBuilder<> &Builder, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg) const override
For some targets, an LLVM struct type must be broken down into multiple simple types,...
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ISD::SETCC ValueType.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool isLegalICmpImmediate(int64_t) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a ldN intrinsic.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
bool fallBackToDAGISel(const Instruction &Inst) const override
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL) const
Returns true if VecTy is a legal interleaved access type.
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
MachineBasicBlock * EmitLoweredCatchRet(MachineInstr &MI, MachineBasicBlock *BB) const
Value * getIRStackGuard(IRBuilder<> &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isProfitableToHoist(Instruction *I) const override
Check if it is profitable to hoist instruction in then/else to if.
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if it is profitable to reduce a load to a smaller type.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a stN intrinsic.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
Value * getSafeStackPointerLocation(IRBuilder<> &IRB) const override
If the target has a standard location for the unsafe stack pointer, returns the address of that locat...
MachineBasicBlock * EmitF128CSEL(MachineInstr &MI, MachineBasicBlock *BB) const
LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &FuncAttributes) const override
LLT returning variant.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool needsFixedCatchObjects() const override
Used for exception handling on Win64.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
AArch64TargetLowering(const TargetMachine &TM, const AArch64Subtarget &STI)
bool isLegalAddImmediate(int64_t) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool shouldConsiderGEPOffsetSplit() const override
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Returns false if N is a bit extraction pattern of (X >> C) & Mask.
bool enableAggressiveFMAFusion(EVT VT) const override
Enable aggressive FMA fusion on targets that want it.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, unsigned Align=1, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, bool *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
Value * emitStoreConditional(IRBuilder<> &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
bool mergeStoresAfterLegalization(EVT VT) const override
SVE code generation for fixed length vectors does not custom lower BUILD_VECTOR.
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
unsigned logBase2() const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
int64_t getSExtValue() const
Get sign extended value.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
bool isFloatingPointOperation() const
BinOp getOperation() const
This is an SDNode representing atomic operations.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Equivalent to hasAttribute(AttributeList::FunctionIndex, Kind) but may be faster.
LLVM Basic Block Representation.
const BlockAddress * getBlockAddress() const
A "pseudo-class" with methods for operating on BUILD_VECTORs.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
static bool resultsCompatible(CallingConv::ID CalleeCC, CallingConv::ID CallerCC, MachineFunction &MF, LLVMContext &C, const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn CalleeFn, CCAssignFn CallerFn)
Returns true if the results of the two calling conventions are compatible.
void AnalyzeCallResult(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeCallResult - Analyze the return values of a call, incorporating info about the passed values i...
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
bool CheckReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
CheckReturn - Analyze the return values of a function, returning true if the return can be performed ...
void AnalyzeReturn(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeReturn - Analyze the returned values of a return, incorporating info about the result values i...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
CCValAssign - Represent assignment of one arg/retval to a location.
Value * getArgOperand(unsigned i) const
unsigned getNumArgOperands() const
This class represents a function call, abstracting a target machine's calling convention.
This is the shared class of boolean and integer constants.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
const APInt & getValue() const
Return the constant as an APInt value reference.
uint64_t getZExtValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
const Function & getFunction() const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
Type * getValueType() const
This instruction inserts a single (scalar) element into a VectorType value.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
This is an important class for using LLVM in a threaded context.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
static ElementCount getScalable(ScalarTy MinVal)
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
This class is used to represent ISD::LOAD nodes.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static mvt_range fixedlen_vector_valuetypes()
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static mvt_range integer_valuetypes()
static mvt_range integer_fixedlen_vector_valuetypes()
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
static mvt_range fp_fixedlen_vector_valuetypes()
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static mvt_range fp_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
void setAdjustsStack(bool V)
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setStackID(int ObjectIdx, uint8_t ID)
bool hasMustTailInVarArgFunc() const
Returns true if the function is variadic and contains a musttail call.
void setReturnAddressIsTaken(bool s)
void computeMaxCallFrameSize(const MachineFunction &MF)
Computes the maximum size of a callframe and the AdjustsStack property.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This is a base class used to represent MGATHER and MSCATTER nodes.
This class is used to represent an MSCATTER node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
unsigned getAlignment() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Class to represent pointers.
Type * getElementType() const
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const TargetSubtargetInfo & getSubtarget() const
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=None, int Offset=0, unsigned TargetFlags=0)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplatValue(SDValue V)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
bool isZeroEltSplat() const
Return true if all elements of this shuffle are the same value as the first element of exactly one so...
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isExtractSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &Index)
Return true if this shuffle mask is an extract subvector mask.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
LLVM_NODISCARD size_t size() const
size - Get the string size.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setTargetDAGCombine(ISD::NodeType NT)
Targets should invoke this method for each target independent node that they want to provide a custom...
virtual void finalizeLowering(MachineFunction &MF) const
Execute target specific actions to finalize target lowering.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual Value * getSafeStackPointerLocation(IRBuilder<> &IRB) const
Returns the target-specific address of the unsafe stack pointer.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual Value * getIRStackGuard(IRBuilder<> &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void setIndexedStoreAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual bool shouldLocalize(const MachineInstr &MI, const TargetTransformInfo *TTI) const
Check whether or not MI needs to be moved close to its uses.
void setMaximumJumpTableSize(unsigned)
Indicate the maximum number of entries in jump tables.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
unsigned getMaximumJumpTableSize() const
Return upper limit for number of entries in a jump table.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool EnableExtLdPromotion
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
virtual void insertSSPDeclarations(Module &M) const
Inserts necessary declarations for SSP (stack protection) purpose.
void setIndexedLoadAction(unsigned IdxMode, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual bool useLoadStackGuardNode() const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
const Triple & getTargetTriple() const
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned EmitCallSiteInfo
The flag enables call site info production.
unsigned TLSSize
Bit size of immediate TLS offsets (0 == use the default).
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned GuaranteedTailCallOpt
GuaranteedTailCallOpt - This flag is enabled when -tailcallopt is specified on the commandline.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Triple - Helper class for working with autoconf configuration names.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isWindowsMSVCEnvironment() const
Checks if the environment could be MSVC.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ScalarTy getFixedSize() const
ScalarTy getKnownMinSize() const
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
@ FloatTyID
32-bit floating point type
@ DoubleTyID
64-bit floating point type
static Type * getVoidTy(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
ScalarTy getValue() const
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
Base class of all SIMD vector types.
Type * getElementType() const
Implementation for an ilist node.
ilist_node_impl()=default
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static CondCode getInvertedCondCode(CondCode Code)
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand, this represents that the reference to the symbol is for an import...
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_HI12
MO_HI12 - This flag indicates that a symbol operand represents the bits 13-24 of a 64-bit address,...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
@ MO_G2
MO_G2 - A symbol operand with this flag (granule 2) represents the bits 32-47 of a 64-bit address,...
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
@ GLDFF1S_SXTW_MERGE_ZERO
@ GLDFF1_SCALED_MERGE_ZERO
@ GLD1_SXTW_SCALED_MERGE_ZERO
@ FP_EXTEND_MERGE_PASSTHRU
@ FP_ROUND_MERGE_PASSTHRU
@ GLDFF1_SXTW_SCALED_MERGE_ZERO
@ UINT_TO_FP_MERGE_PASSTHRU
@ FROUNDEVEN_MERGE_PASSTHRU
@ GLD1S_UXTW_SCALED_MERGE_ZERO
@ GLDNT1_INDEX_MERGE_ZERO
@ GLDFF1_UXTW_SCALED_MERGE_ZERO
@ FNEARBYINT_MERGE_PASSTHRU
@ GLDFF1S_SCALED_MERGE_ZERO
@ GLDFF1S_UXTW_SCALED_MERGE_ZERO
@ ZERO_EXTEND_INREG_MERGE_PASSTHRU
@ NVCAST
Natural vector cast.
@ BITREVERSE_MERGE_PASSTHRU
@ GLDFF1S_UXTW_MERGE_ZERO
@ SIGN_EXTEND_INREG_MERGE_PASSTHRU
@ GLDFF1S_SXTW_SCALED_MERGE_ZERO
@ GLD1S_SCALED_MERGE_ZERO
@ SINT_TO_FP_MERGE_PASSTHRU
@ GLD1_UXTW_SCALED_MERGE_ZERO
@ GLD1S_SXTW_SCALED_MERGE_ZERO
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint8_t encodeAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType9(uint64_t Imm)
static bool isAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType5(uint64_t Imm)
static int getFP32Imm(const APInt &Imm)
getFP32Imm - Return an 8-bit floating-point version of the 32-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType10(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType9(uint64_t Imm)
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static bool isAdvSIMDModImmType7(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType5(uint64_t Imm)
static int getFP64Imm(const APInt &Imm)
getFP64Imm - Return an 8-bit floating-point version of the 64-bit floating-point value.
static bool isAdvSIMDModImmType10(uint64_t Imm)
static int getFP16Imm(const APInt &Imm)
getFP16Imm - Return an 8-bit floating-point version of the 16-bit floating-point value.
static uint8_t encodeAdvSIMDModImmType8(uint64_t Imm)
static bool isAdvSIMDModImmType12(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType11(uint64_t Imm)
static bool isAdvSIMDModImmType11(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType6(uint64_t Imm)
static bool isAdvSIMDModImmType8(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType4(uint64_t Imm)
static bool isAdvSIMDModImmType6(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType1(uint64_t Imm)
static uint8_t encodeAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType2(uint64_t Imm)
static bool isAdvSIMDModImmType3(uint64_t Imm)
static bool isAdvSIMDModImmType1(uint64_t Imm)
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static constexpr unsigned SVEBitsPerBlock
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ AArch64_SVE_VectorCall
Calling convention between AArch64 SVE functions.
@ CFGuard_Check
Special calling convention on Windows for calling the Control Guard Check ICall funtion.
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
C - The default llvm calling convention, compatible with C.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ FLT_ROUNDS_
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isOverflowIntrOpRes(SDValue Op)
Returns true if the specified value is the overflow result from one of the overflow intrinsic nodes.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
static const int LAST_INDEXED_MODE
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
bool match(Val *V, const Pattern &P)
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
class_match< UndefValue > m_Undef()
Match an arbitrary undef constant.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
match_combine_or< CastClass_match< OpTy, Instruction::ZExt >, CastClass_match< OpTy, Instruction::SExt > > m_ZExtOrSExt(const OpTy &Op)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
initializer< Ty > init(const Ty &Val)
CodeModel::Model getCodeModel()
This class represents lattice values for constants.
bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool operator==(uint64_t V1, const APInt &V2)
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
testing::Matcher< const detail::ErrorHolder & > Failed()
bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
void shuffle(Iterator first, Iterator last, RNG &&g)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Z
zlib style complession
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
@ Invalid
Denotes invalid value.
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
bool isAcquireOrStronger(AtomicOrdering AO)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr unsigned BitWidth
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
gep_type_iterator gep_type_begin(const User *GEP)
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
bool CC_AArch64_DarwinPCS_ILP32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
static const MachineMemOperand::Flags MOStridedAccess
bool CC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Helper structure to keep track of a SET_CC lowered into AArch64 code.
Helper structure to keep track of ISD::SET_CC operands.
Helper structure to be able to read SetCC information.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
EVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
bool isFixedLengthVector() const
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Describes a register that needs to be forwarded from the prologue to a musttail call.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
static KnownBits commonBits(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits common to LHS and RHS.
unsigned getBitWidth() const
Get the bit width of this value.
Structure used to represent pair of argument number after call lowering and register used to transfer...
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op is the value the constraint a...
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowReassociation(bool b)
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
This structure contains all information that is necessary for lowering calls.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
Helper structure to keep track of SetCC information.